Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.32809883 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.03180336 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.03585545 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1188 1189 1190 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.474080621Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.545891402Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.54873005Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.549275035Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.549312912Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.549343284Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"Crucible afa604e7-efed-451e-bb36-ca54c895e52b has session id: 1c003785-e56c-46e8-a611-0a2399bbb846","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.551083785Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Upstairs opts: Upstairs UUID: afa604e7-efed-451e-bb36-ca54c895e52b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.551401095Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Crucible stats registered with UUID: afa604e7-efed-451e-bb36-ca54c895e52b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.551424382Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:50.551439543Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.553714177Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.553816997Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"afa604e7-efed-451e-bb36-ca54c895e52b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.553853232Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.553934177Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554430342Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554485285Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554505837Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554522761Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554541179Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554557222Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554573946Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.554590149Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"ds_connection connected from Ok(127.0.0.1:48178)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.555004988Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"ds_connection connected from Ok(127.0.0.1:38257)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.555029659Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"ds_connection connected from Ok(127.0.0.1:46984)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.55538736Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.557803464Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.557849047Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.557875341Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.55955139Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.559587253Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.560081785Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.56120332Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561239033Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561336372Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561363597Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561385412Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561404451Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.56142359Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.56144289Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561459744Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561476318Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561501149Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561520699Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561539327Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561557214Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.56157517Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"downstairs","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561593508Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.56160933Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561626004Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561642538Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"client":"2","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561658561Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561673261Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"afa604e7-efed-451e-bb36-ca54c895e52b is now active with session: 1c003785-e56c-46e8-a611-0a2399bbb846","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.561776011Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:02:50.561798898Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:02:50.562214098Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"Activated sub_volume afa604e7-efed-451e-bb36-ca54c895e52b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:50.562296376Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663017953Z","hostname":"bmat-EVT22200007-0000589c","pid":1193} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663113879Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663155795Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663187911Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.663217822Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.663247222Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663283737Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.663313798Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.663342156Z","hostname":"bmat-EVT22200007-0000589c","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1c003785-e56c-46e8-a611-0a2399bbb846"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.705607399Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.756720514Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.757093768Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.794141435Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.794786004Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.794830404Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.794848632Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.794866518Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"Crucible 7cf03d66-f46e-48dd-a479-2713f1f1edf2 has session id: 422e4750-05a2-44fa-8f92-7c9c45212cf1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.795970146Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Upstairs opts: Upstairs UUID: 7cf03d66-f46e-48dd-a479-2713f1f1edf2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796020159Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Crucible stats registered with UUID: 7cf03d66-f46e-48dd-a479-2713f1f1edf2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796039328Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.796054309Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.79674889Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796778129Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"7cf03d66-f46e-48dd-a479-2713f1f1edf2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796897423Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796917825Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796947546Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.79696419Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.796987057Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797003911Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797089196Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.79710571Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797122334Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797139118Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44523)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797347384Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62928)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797520128Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35111)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.797923042Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.79840596Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.798731326Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.798788042Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.799360894Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.799391727Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.79946171Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800142202Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800250604Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800393176Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800420762Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800438899Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800455433Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800471576Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800488491Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800504233Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800520015Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800609779Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800634249Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800650532Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800725015Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:02:54.800891615Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"":"downstairs","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800927098Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800941918Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"0","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.80095762Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"1","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.800973473Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"client":"2","":"downstairs","session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.801060451Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.801175857Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"7cf03d66-f46e-48dd-a479-2713f1f1edf2 is now active with session: 422e4750-05a2-44fa-8f92-7c9c45212cf1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.80119205Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:02:54.801317868Z","hostname":"bmat-EVT22200007-0000589c","pid":1195,"session_id":"422e4750-05a2-44fa-8f92-7c9c45212cf1"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.801344061Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} {"msg":"Activated sub_volume 7cf03d66-f46e-48dd-a479-2713f1f1edf2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:54.80135805Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 357 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Write block 570 len 24576 data: 2 2 2 2 2 2 03/30 Write block 529 len 40960 data: 2 2 2 2 2 2 2 2 2 2 04/30 Write block 100 len 4096 data: 2 05/30 Flush 06/30 Write block 418 len 12288 data: 2 2 2 07/30 Read block 102 len 28672 08/30 Write block 222 len 20480 data: 2 2 2 2 2 09/30 Read block 120 len 36864 10/30 Write block 463 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Read block 206 len 24576 12/30 Read block 71 len 4096 13/30 Write block 278 len 40960 data: 2 2 2 2 2 2 2 2 2 2 14/30 Read block 230 len 20480 15/30 Read block 85 len 12288 16/30 Write block 9 len 8192 data: 2 2 17/30 Read block 182 len 28672 18/30 Read block 1 len 32768 19/30 Read block 294 len 32768 20/30 Flush 21/30 Write block 34 len 16384 data: 2 2 2 2 22/30 Read block 236 len 8192 23/30 Read block 311 len 16384 24/30 Read block 186 len 28672 25/30 Read block 452 len 16384 26/30 Write block 400 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Read block 583 len 4096 28/30 Write block 461 len 40960 data: 2 2 3 3 3 3 3 3 3 3 29/30 Read block 87 len 12288 30/30 Write block 137 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:54.831227752Z","hostname":"bmat-EVT22200007-0000589c","pid":1195} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 1 1 1 1 1 1 T T T 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 2 2 2 3 3 3 T T T 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 3 3 3 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 3 3 3 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 3 3 3 4 4 4 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.193511744Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.247276704Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.248031849Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.248088876Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.248106863Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.248124248Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"Crucible 6c13def8-492e-424b-a058-2a7961cd5d86 has session id: 6deee74e-b5dc-44eb-bbd5-584c55ae743c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.249280063Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Upstairs opts: Upstairs UUID: 6c13def8-492e-424b-a058-2a7961cd5d86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.249328031Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Crucible stats registered with UUID: 6c13def8-492e-424b-a058-2a7961cd5d86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.249348934Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:59.249364255Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250039888Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250069999Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"6c13def8-492e-424b-a058-2a7961cd5d86 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250186037Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250208322Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250238714Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250255379Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250272544Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250288657Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250305902Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250321624Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250788099Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250815755Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44289)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250833682Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52565)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250850165Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46568)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.250866298Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.252022845Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.252059159Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.252077396Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.252859697Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.25289519Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.252931624Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.253922882Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.253987484Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[0]R flush_numbers[0..12]: [3, 4, 1, 1, 1, 2, 4, 4, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254005742Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[0]R generation[0..12]: [2, 3, 1, 1, 1, 2, 3, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254025652Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254043609Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[1]R flush_numbers[0..12]: [3, 4, 1, 1, 1, 2, 4, 4, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254060073Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[1]R generation[0..12]: [2, 3, 1, 1, 1, 2, 3, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254076146Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.25409301Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[2]R flush_numbers[0..12]: [3, 4, 1, 1, 1, 2, 4, 4, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254109885Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[2]R generation[0..12]: [2, 3, 1, 1, 1, 2, 3, 3, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254135086Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254152712Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254168314Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254184026Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254199348Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254214709Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"":"downstairs","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254230451Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254244971Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"0","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254261405Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"1","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254277347Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"client":"2","":"downstairs","session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.25429319Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254307639Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"6c13def8-492e-424b-a058-2a7961cd5d86 is now active with session: 6deee74e-b5dc-44eb-bbd5-584c55ae743c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254323201Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.25433746Z","hostname":"bmat-EVT22200007-0000589c","pid":1203,"session_id":"6deee74e-b5dc-44eb-bbd5-584c55ae743c"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254352311Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} {"msg":"Activated sub_volume 6c13def8-492e-424b-a058-2a7961cd5d86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:02:59.254366219Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 384 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 431 len 24576 data: 2 2 2 2 2 2 03/30 Write block 581 len 24576 data: 2 2 2 2 2 2 04/30 Read block 440 len 32768 05/30 Read block 137 len 20480 06/30 Read block 591 len 36864 07/30 Read block 258 len 4096 08/30 Write block 306 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Write block 432 len 16384 data: 3 3 3 3 10/30 Write block 273 len 40960 data: 2 2 2 2 2 3 3 3 3 3 11/30 Write block 206 len 20480 data: 2 2 2 2 2 12/30 Read block 273 len 24576 13/30 Write block 251 len 8192 data: 2 2 14/30 Read block 138 len 36864 15/30 Write block 585 len 40960 data: 3 3 2 2 2 2 2 2 2 2 16/30 Write block 219 len 28672 data: 2 2 2 3 3 3 3 17/30 Read block 545 len 8192 18/30 Write block 298 len 36864 data: 2 2 2 2 2 2 2 2 3 19/30 Read block 218 len 12288 20/30 Write block 46 len 28672 data: 2 2 2 2 2 2 2 21/30 Read block 584 len 28672 22/30 Write block 436 len 28672 data: 3 2 2 2 2 2 2 23/30 Write block 286 len 8192 data: 3 3 24/30 Write block 78 len 20480 data: 2 2 2 2 2 25/30 Write block 477 len 20480 data: 2 2 2 2 2 26/30 Write block 465 len 24576 data: 4 4 4 4 4 4 27/30 Write block 210 len 32768 data: 3 2 2 2 2 2 2 2 28/30 Write block 231 len 8192 data: 2 2 29/30 Read block 254 len 12288 30/30 Read block 509 len 12288 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:02:59.278757543Z","hostname":"bmat-EVT22200007-0000589c","pid":1203} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 T T T 5 100-119 2 2 2 2 2 2 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 2 2 2 3 3 3 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 2 2 2 3 3 3 T T T 14 280-299 2 2 2 3 3 3 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 3 3 3 4 4 4 F F F 21 420-439 2 2 2 3 3 3 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 3 3 3 4 4 4 T T T 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 3, Max flush: 4 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 4 4 4 5 5 5 F F F 4 080-099 4 4 4 5 5 5 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 4 4 4 5 5 5 F F F 11 220-239 4 4 4 5 5 5 F F F 12 240-259 4 4 4 5 5 5 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 4 4 4 5 5 5 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 3 3 3 4 4 4 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 4, Max flush: 5 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.662038392Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.714545369Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.715252315Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.715311106Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.715330195Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.715348602Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"Crucible 22fd8d58-bc63-4be3-bda5-bd5726166fc6 has session id: 9b604067-8d96-4e89-9650-b0b94f3040bb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.716501571Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Upstairs opts: Upstairs UUID: 22fd8d58-bc63-4be3-bda5-bd5726166fc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.716552766Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Crucible stats registered with UUID: 22fd8d58-bc63-4be3-bda5-bd5726166fc6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.716571955Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:03.716587377Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717303883Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717335217Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"22fd8d58-bc63-4be3-bda5-bd5726166fc6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717472989Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717500034Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717842014Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717865502Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.717891335Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.7180061Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718083618Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718359182Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718387921Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718409445Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43752)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718431209Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37998)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718452663Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41038)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.718473716Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.71909616Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.719320419Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.719882148Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.719984547Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.720688397Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.720719792Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.721174622Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.721881058Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[0]R flush_numbers[0..12]: [3, 4, 5, 5, 5, 2, 4, 4, 1, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722645983Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[0]R generation[0..12]: [2, 3, 4, 4, 4, 2, 3, 3, 1, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.72268883Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722711557Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[1]R flush_numbers[0..12]: [3, 4, 5, 5, 5, 2, 4, 4, 1, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722731618Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[1]R generation[0..12]: [2, 3, 4, 4, 4, 2, 3, 3, 1, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722751017Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722771179Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[2]R flush_numbers[0..12]: [3, 4, 5, 5, 5, 2, 4, 4, 1, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722790618Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[2]R generation[0..12]: [2, 3, 4, 4, 4, 2, 3, 3, 1, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722809517Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722829759Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Max found gen is 5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722848437Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Generation requested: 6 >= found:5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722867035Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722885282Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722909852Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"":"downstairs","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722934072Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722951738Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"0","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.722970767Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"1","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.72299174Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"client":"2","":"downstairs","session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.72301123Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.723028846Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Wait for a query_work_queue command to finish before sending IO 22fd8d58-bc63-4be3-bda5-bd5726166fc6 is now active with session: 9b604067-8d96-4e89-9650-b0b94f3040bb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.723062525Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.723080902Z","hostname":"bmat-EVT22200007-0000589c","pid":1211,"session_id":"9b604067-8d96-4e89-9650-b0b94f3040bb"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.723098088Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} {"msg":"Activated sub_volume 22fd8d58-bc63-4be3-bda5-bd5726166fc6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:03.723114251Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 16384 data: 2 2 2 2 02/30 Flush 03/30 Write block 217 len 40960 data: 3 2 3 3 3 4 4 4 4 3 04/30 Read block 83 len 20480 05/30 Write block 99 len 32768 data: 2 3 2 2 2 2 2 2 06/30 Read block 419 len 36864 07/30 Read block 543 len 32768 08/30 Write block 321 len 8192 data: 2 2 09/30 Read block 118 len 40960 10/30 Write block 139 len 32768 data: 3 3 3 3 3 3 3 3 11/30 Read block 453 len 8192 12/30 Write block 53 len 8192 data: 2 2 13/30 Read block 56 len 12288 14/30 Read block 326 len 20480 15/30 Write block 373 len 16384 data: 2 2 2 2 16/30 Read block 451 len 36864 17/30 Flush 18/30 Write block 373 len 16384 data: 3 3 3 3 19/30 Read block 302 len 28672 20/30 Read block 534 len 16384 21/30 Read block 5 len 20480 22/30 Flush 23/30 Write block 316 len 20480 data: 2 2 2 2 2 24/30 Write block 333 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 403 len 32768 data: 3 3 3 3 3 3 3 2 26/30 Read block 309 len 12288 27/30 Read block 208 len 20480 28/30 Read block 292 len 4096 29/30 Write block 557 len 4096 data: 2 30/30 Read block 16 len 16384 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:03.751214372Z","hostname":"bmat-EVT22200007-0000589c","pid":1211} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 5 5 5 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 4 4 4 5 5 5 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 4 4 4 5 5 5 T T T 16 320-339 6 6 6 7 7 7 T T T 17 340-359 2 2 2 2 2 2 T T T 18 360-379 6 6 6 8 8 8 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 3 3 3 4 4 4 T T T 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 6, Max flush: 8 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 5 5 5 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 4 4 4 5 5 5 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 7 7 7 9 9 9 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 F F F 18 360-379 6 6 6 8 8 8 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 7 7 7 9 9 9 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 7 7 7 9 9 9 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 7, Max flush: 9 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.068305261Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.122032945Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.122827781Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.122885078Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.122903596Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.122921523Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"Crucible 8f45612b-98b5-48d3-b1f8-c3bbf6426638 has session id: d6e507ff-5b92-436c-a21d-59dcaca2fc5f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.123825703Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Upstairs opts: Upstairs UUID: 8f45612b-98b5-48d3-b1f8-c3bbf6426638, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.123873641Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Crucible stats registered with UUID: 8f45612b-98b5-48d3-b1f8-c3bbf6426638","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.123892249Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:08.1239076Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.124574605Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.124612372Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"8f45612b-98b5-48d3-b1f8-c3bbf6426638 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.124787851Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.124812381Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125357417Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125591255Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125630996Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125649935Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125668002Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125684315Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125701901Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125718646Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49097)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.12573559Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49053)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125752515Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40338)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.125769339Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.126421654Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.12644886Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.126472157Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.127403994Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.127689197Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.127745793Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128340209Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128418469Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[0]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 7, 7, 7, 1, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128554247Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[0]R generation[0..12]: [2, 3, 6, 4, 6, 6, 6, 6, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128579749Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.12859982Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[1]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 7, 7, 7, 1, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128618729Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[1]R generation[0..12]: [2, 3, 6, 4, 6, 6, 6, 6, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128637427Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128655935Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[2]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 7, 7, 7, 1, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128672429Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[2]R generation[0..12]: [2, 3, 6, 4, 6, 6, 6, 6, 1, 1, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128688341Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128769247Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128791793Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128809399Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128825081Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128840312Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"":"downstairs","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:03:08.128929044Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128955027Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"0","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.12897123Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"1","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.128987143Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"client":"2","":"downstairs","session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129003136Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129018337Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"8f45612b-98b5-48d3-b1f8-c3bbf6426638 is now active with session: d6e507ff-5b92-436c-a21d-59dcaca2fc5f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129095144Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129109994Z","hostname":"bmat-EVT22200007-0000589c","pid":1219,"session_id":"d6e507ff-5b92-436c-a21d-59dcaca2fc5f"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129124664Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} {"msg":"Activated sub_volume 8f45612b-98b5-48d3-b1f8-c3bbf6426638","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:08.129137741Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 364 len 32768 data: 3 3 3 2 2 2 2 2 02/30 Write block 436 len 12288 data: 4 3 3 03/30 Read block 272 len 36864 04/30 Read block 268 len 40960 05/30 Read block 10 len 16384 06/30 Flush 07/30 Write block 256 len 16384 data: 2 2 2 2 08/30 Read block 3 len 28672 09/30 Write block 566 len 12288 data: 2 2 2 10/30 Read block 23 len 32768 11/30 Read block 31 len 36864 12/30 Write block 566 len 12288 data: 3 3 3 13/30 Write block 567 len 40960 data: 4 4 2 3 3 3 3 3 3 2 14/30 Write block 108 len 4096 data: 2 15/30 Read block 165 len 12288 16/30 Read block 243 len 8192 17/30 Write block 459 len 24576 data: 2 2 3 3 4 4 18/30 Write block 233 len 24576 data: 2 2 2 2 2 2 19/30 Read block 518 len 20480 20/30 Read block 546 len 4096 21/30 Write block 112 len 24576 data: 2 2 2 2 2 2 22/30 Read block 492 len 20480 23/30 Write block 456 len 12288 data: 2 2 2 24/30 Write block 430 len 36864 data: 2 3 4 4 4 4 5 4 4 25/30 Read block 228 len 40960 26/30 Write block 528 len 20480 data: 2 3 3 3 3 27/30 Write block 534 len 32768 data: 3 3 3 3 3 2 2 2 28/30 Write block 387 len 40960 data: 3 3 3 3 3 3 2 2 2 2 29/30 Write block 544 len 4096 data: 2 30/30 Write block 295 len 36864 data: 2 2 2 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:08.155717532Z","hostname":"bmat-EVT22200007-0000589c","pid":1219} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 5 5 5 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 6 6 6 7 7 7 T T T 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 T T T 12 240-259 4 4 4 5 5 5 T T T 13 260-279 4 4 4 5 5 5 F F F 14 280-299 4 4 4 5 5 5 T T T 15 300-319 7 7 7 9 9 9 T T T 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 4 4 4 5 5 5 T T T 20 400-419 7 7 7 9 9 9 F F F 21 420-439 8 8 8 10 10 10 T T T 22 440-459 4 4 4 5 5 5 T T T 23 460-479 4 4 4 5 5 5 T T T 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 T T T 27 540-559 7 7 7 9 9 9 T T T 28 560-579 2 2 2 2 2 2 T T T 29 580-599 4 4 4 5 5 5 F F F Max gen: 8, Max flush: 10 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 7 7 7 F F F 3 060-079 4 4 4 5 5 5 F F F 4 080-099 6 6 6 7 7 7 F F F 5 100-119 9 9 9 11 11 11 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 9 9 9 11 11 11 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 F F F 18 360-379 8 8 8 10 10 10 F F F 19 380-399 9 9 9 11 11 11 F F F 20 400-419 7 7 7 9 9 9 F F F 21 420-439 9 9 9 11 11 11 F F F 22 440-459 9 9 9 11 11 11 F F F 23 460-479 9 9 9 11 11 11 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F F F 28 560-579 9 9 9 11 11 11 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 9, Max flush: 11 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.517140633Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:12.568080463Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.568483708Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.604702039Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.605437273Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.60551902Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.605546948Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.605580556Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"Crucible 30b994eb-007d-4496-ab02-058a740a8866 has session id: c2b95abf-2f7c-4b81-8596-989d191cc4d7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.606385674Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Upstairs opts: Upstairs UUID: 30b994eb-007d-4496-ab02-058a740a8866, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.606541633Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Crucible stats registered with UUID: 30b994eb-007d-4496-ab02-058a740a8866","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.60657432Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:12.606591194Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.607280214Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.60731277Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"30b994eb-007d-4496-ab02-058a740a8866 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.60742965Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.607734924Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.607808946Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.607828806Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.607846392Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608181708Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608311524Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608486832Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608609473Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608634855Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56108)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608652772Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49149)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608669867Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48718)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.608686681Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.609112913Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.609185051Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.609206274Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.609675324Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.609706067Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610092588Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610781457Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610857463Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[0]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 11, 7, 7, 1, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610887204Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[0]R generation[0..12]: [2, 3, 6, 4, 6, 9, 6, 6, 1, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610918838Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.61094888Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[1]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 11, 7, 7, 1, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.610977709Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[1]R generation[0..12]: [2, 3, 6, 4, 6, 9, 6, 6, 1, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611006187Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611034826Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[2]R flush_numbers[0..12]: [3, 4, 7, 5, 7, 11, 7, 7, 1, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611050838Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[2]R generation[0..12]: [2, 3, 6, 4, 6, 9, 6, 6, 1, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611066651Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611082854Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611097975Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:03:12.611113537Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611222199Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611241358Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"":"downstairs","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611262521Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.61138395Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"0","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611408059Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"1","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611424463Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"client":"2","":"downstairs","session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:03:12.611637559Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611660025Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"30b994eb-007d-4496-ab02-058a740a8866 is now active with session: c2b95abf-2f7c-4b81-8596-989d191cc4d7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611677531Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.61169197Z","hostname":"bmat-EVT22200007-0000589c","pid":1227,"session_id":"c2b95abf-2f7c-4b81-8596-989d191cc4d7"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.61170648Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} {"msg":"Activated sub_volume 30b994eb-007d-4496-ab02-058a740a8866","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:12.611719797Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 8192 data: 2 2 02/30 Read block 305 len 8192 03/30 Write block 75 len 24576 data: 2 2 2 3 3 3 04/30 Read block 375 len 36864 05/30 Flush 06/30 Write block 26 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 587 len 12288 08/30 Read block 5 len 20480 09/30 Read block 508 len 32768 10/30 Write block 556 len 16384 data: 2 3 2 2 11/30 Write block 383 len 4096 data: 2 12/30 Read block 272 len 40960 13/30 Write block 52 len 20480 data: 3 3 3 2 2 14/30 Write block 460 len 24576 data: 3 4 4 5 5 5 15/30 Read block 126 len 4096 16/30 Flush 17/30 Write block 556 len 40960 data: 3 4 3 3 2 2 2 2 2 2 18/30 Read block 317 len 20480 19/30 Read block 61 len 16384 20/30 Read block 454 len 8192 21/30 Read block 185 len 4096 22/30 Write block 369 len 40960 data: 3 3 3 2 4 4 4 4 2 2 23/30 Read block 460 len 4096 24/30 Read block 260 len 32768 25/30 Read block 543 len 4096 26/30 Read block 89 len 36864 27/30 Read block 328 len 24576 28/30 Read block 506 len 4096 29/30 Read block 443 len 32768 30/30 Write block 288 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:12.638972224Z","hostname":"bmat-EVT22200007-0000589c","pid":1227} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 10 10 10 13 13 13 F F F 2 040-059 10 10 10 13 13 13 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 9 9 9 11 11 11 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 10 10 10 12 12 12 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 9 9 9 11 11 11 T T T 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 F F F 18 360-379 8 8 8 10 10 10 T T T 19 380-399 10 10 10 13 13 13 F F F 20 400-419 7 7 7 9 9 9 F F F 21 420-439 9 9 9 11 11 11 F F F 22 440-459 9 9 9 11 11 11 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 10 10 10 13 13 13 T T T 28 560-579 9 9 9 11 11 11 T T T 29 580-599 4 4 4 5 5 5 F F F Max gen: 10, Max flush: 13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 10 10 10 13 13 13 F F F 2 040-059 10 10 10 13 13 13 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 9 9 9 11 11 11 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 10 10 10 12 12 12 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 4 4 4 5 5 5 F F F 14 280-299 11 11 11 14 14 14 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 F F F 18 360-379 11 11 11 14 14 14 F F F 19 380-399 10 10 10 13 13 13 F F F 20 400-419 7 7 7 9 9 9 F F F 21 420-439 9 9 9 11 11 11 F F F 22 440-459 9 9 9 11 11 11 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 14 14 14 F F F 28 560-579 11 11 11 14 14 14 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 11, Max flush: 14 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:16.98237257Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:17.036005962Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.036452696Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.073620269Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.074628652Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.074681881Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.074702072Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.07475969Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"Crucible f775d99d-ce5c-4eb7-80c3-e95cdc1ff915 has session id: f539de66-df9e-4042-bfd1-05e6ea26ad28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.075645973Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Upstairs opts: Upstairs UUID: f775d99d-ce5c-4eb7-80c3-e95cdc1ff915, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.075822274Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Crucible stats registered with UUID: f775d99d-ce5c-4eb7-80c3-e95cdc1ff915","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.075848487Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:17.075871224Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076591808Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076629876Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"f775d99d-ce5c-4eb7-80c3-e95cdc1ff915 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.07676356Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076787118Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076817861Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076834725Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.0768519Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076867973Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076885128Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076966304Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.076985895Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.077004232Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"ds_connection connected from Ok(127.0.0.1:46928)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.077157115Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"ds_connection connected from Ok(127.0.0.1:51119)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.077180112Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"ds_connection connected from Ok(127.0.0.1:37599)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.077421636Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.078432735Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.078494231Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.078514041Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.07937377Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.079522043Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.079547746Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080213328Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080242347Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[0]R flush_numbers[0..12]: [3, 13, 13, 12, 12, 11, 7, 7, 12, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080359627Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[0]R generation[0..12]: [2, 10, 10, 10, 10, 9, 6, 6, 10, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080383897Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080402765Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[1]R flush_numbers[0..12]: [3, 13, 13, 12, 12, 11, 7, 7, 12, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080420812Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[1]R generation[0..12]: [2, 10, 10, 10, 10, 9, 6, 6, 10, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080438498Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080456746Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[2]R flush_numbers[0..12]: [3, 13, 13, 12, 12, 11, 7, 7, 12, 1, 7, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080479162Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[2]R generation[0..12]: [2, 10, 10, 10, 10, 9, 6, 6, 10, 1, 6, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080496778Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080514845Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080531489Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080548093Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080564476Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080580749Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"":"downstairs","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080597363Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080612675Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"0","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080629219Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"1","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080645843Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"client":"2","":"downstairs","session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080662827Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080680103Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"f775d99d-ce5c-4eb7-80c3-e95cdc1ff915 is now active with session: f539de66-df9e-4042-bfd1-05e6ea26ad28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080696857Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"Set Active after no reconciliation","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080723752Z","hostname":"bmat-EVT22200007-0000589c","pid":1235,"session_id":"f539de66-df9e-4042-bfd1-05e6ea26ad28"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080740857Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} {"msg":"Activated sub_volume f775d99d-ce5c-4eb7-80c3-e95cdc1ff915","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:17.080760678Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 28672 data: 4 4 4 4 4 4 4 02/30 Write block 391 len 16384 data: 4 4 3 3 03/30 Read block 377 len 36864 04/30 Write block 411 len 20480 data: 2 2 2 2 2 05/30 Write block 265 len 32768 data: 2 2 2 2 2 2 2 2 06/30 Read block 341 len 4096 07/30 Flush 08/30 Write block 412 len 36864 data: 3 3 3 3 2 2 3 3 3 09/30 Read block 27 len 36864 10/30 Write block 400 len 20480 data: 3 3 3 4 4 11/30 Read block 456 len 40960 12/30 Write block 105 len 24576 data: 3 3 2 3 2 2 13/30 Read block 296 len 36864 14/30 Read block 251 len 28672 15/30 Read block 115 len 8192 16/30 Read block 38 len 32768 17/30 Read block 110 len 20480 18/30 Write block 371 len 16384 data: 4 3 5 5 19/30 Write block 155 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Write block 203 len 4096 data: 2 21/30 Read block 456 len 40960 22/30 Write block 102 len 32768 data: 3 3 3 4 4 3 4 3 23/30 Write block 342 len 28672 data: 2 2 2 2 2 2 2 24/30 Write block 47 len 4096 data: 3 25/30 Write block 469 len 20480 data: 5 5 3 2 2 26/30 Read block 10 len 40960 27/30 Read block 187 len 24576 28/30 Read block 590 len 32768 29/30 Write block 442 len 24576 data: 3 2 2 2 2 2 30/30 Read block 288 len 4096 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 9 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 23 21 21 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:17.107968674Z","hostname":"bmat-EVT22200007-0000589c","pid":1235} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 10 10 10 13 13 13 F F F 2 040-059 10 10 10 13 13 13 T T T 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 9 9 9 11 11 11 T T T 6 120-139 6 6 6 7 7 7 F F F 7 140-159 12 12 12 15 15 15 T T T 8 160-179 10 10 10 12 12 12 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 7 7 7 T T T 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 11 11 11 14 14 14 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 7 7 7 9 9 9 T T T 18 360-379 11 11 11 14 14 14 T T T 19 380-399 12 12 12 15 15 15 F F F 20 400-419 12 12 12 15 15 15 T T T 21 420-439 9 9 9 11 11 11 T T T 22 440-459 9 9 9 11 11 11 T T T 23 460-479 10 10 10 13 13 13 T T T 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 14 14 14 F F F 28 560-579 11 11 11 14 14 14 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 12, Max flush: 15 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 10 10 10 13 13 13 F F F 2 040-059 13 13 13 16 16 16 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 13 13 13 16 16 16 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 13 13 13 16 16 16 F F F 8 160-179 13 13 13 16 16 16 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 13 13 13 16 16 16 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 11 11 11 14 14 14 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 13 13 13 16 16 16 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 13 13 13 16 16 16 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 13 13 13 16 16 16 F F F 23 460-479 13 13 13 16 16 16 F F F 24 480-499 4 4 4 5 5 5 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 14 14 14 F F F 28 560-579 11 11 11 14 14 14 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 13, Max flush: 16 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.500311554Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:21.549807119Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.550251699Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.586314261Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.586989763Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.587049956Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.587076551Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.58710568Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"Crucible c4ae2c87-35b9-43fc-836f-3474e8cd4775 has session id: e75848f1-bba5-4ab1-8a9d-0ab42716cca3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.588002836Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Upstairs opts: Upstairs UUID: c4ae2c87-35b9-43fc-836f-3474e8cd4775, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.588075284Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Crucible stats registered with UUID: c4ae2c87-35b9-43fc-836f-3474e8cd4775","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.588097239Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:21.588113001Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.588914962Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.588945775Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"c4ae2c87-35b9-43fc-836f-3474e8cd4775 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.58906077Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589081934Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589111153Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589135634Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589154081Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589170034Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589187039Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589202751Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589219495Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589235798Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41693)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589812679Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37476)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.589854925Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41073)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.590263922Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.590611233Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.591016512Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.591060842Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.591443906Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.591468567Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.591603613Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592145842Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592262721Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[0]R flush_numbers[0..12]: [3, 13, 16, 12, 12, 16, 7, 16, 16, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592424763Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[0]R generation[0..12]: [2, 10, 13, 10, 10, 13, 6, 13, 13, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592448762Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592466138Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[1]R flush_numbers[0..12]: [3, 13, 16, 12, 12, 16, 7, 16, 16, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592483243Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[1]R generation[0..12]: [2, 10, 13, 10, 10, 13, 6, 13, 13, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592499756Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592516631Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[2]R flush_numbers[0..12]: [3, 13, 16, 12, 12, 16, 7, 16, 16, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592535099Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[2]R generation[0..12]: [2, 10, 13, 10, 10, 13, 6, 13, 13, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592551683Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592568186Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592583668Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.59259946Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592621155Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592637007Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"":"downstairs","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592652739Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592667419Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"0","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592683492Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"1","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592699214Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"client":"2","":"downstairs","session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592715167Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592729526Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"c4ae2c87-35b9-43fc-836f-3474e8cd4775 is now active with session: e75848f1-bba5-4ab1-8a9d-0ab42716cca3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592745028Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592759137Z","hostname":"bmat-EVT22200007-0000589c","pid":1243,"session_id":"e75848f1-bba5-4ab1-8a9d-0ab42716cca3"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592773486Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} {"msg":"Activated sub_volume c4ae2c87-35b9-43fc-836f-3474e8cd4775","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:21.592787375Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 526 len 24576 data: 2 2 3 4 4 4 02/30 Write block 42 len 12288 data: 3 3 2 03/30 Flush 04/30 Write block 104 len 4096 data: 4 05/30 Read block 496 len 8192 06/30 Read block 271 len 24576 07/30 Flush 08/30 Write block 344 len 4096 data: 3 09/30 Read block 21 len 20480 10/30 Write block 143 len 8192 data: 5 5 11/30 Write block 513 len 24576 data: 2 2 2 2 2 2 12/30 Read block 345 len 12288 13/30 Write block 24 len 20480 data: 2 2 3 3 3 14/30 Write block 576 len 40960 data: 3 2 2 2 2 3 3 3 3 4 15/30 Write block 520 len 40960 data: 2 2 2 2 2 2 3 3 4 5 16/30 Write block 12 len 4096 data: 2 17/30 Read block 124 len 4096 18/30 Write block 495 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Write block 404 len 8192 data: 5 4 20/30 Read block 433 len 28672 21/30 Flush 22/30 Write block 500 len 36864 data: 3 3 3 2 2 2 2 2 2 23/30 Read block 100 len 24576 24/30 Write block 22 len 24576 data: 2 2 3 3 4 4 25/30 Write block 478 len 40960 data: 3 3 3 3 2 2 2 2 2 2 26/30 Write block 131 len 4096 data: 2 27/30 Read block 236 len 36864 28/30 Write block 167 len 40960 data: 2 2 2 2 2 2 2 2 2 2 29/30 Write block 167 len 32768 data: 3 3 3 3 3 3 3 3 30/30 Read block 504 len 32768 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 7 9 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:21.617818297Z","hostname":"bmat-EVT22200007-0000589c","pid":1243} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 14 14 14 19 19 19 T T T 2 040-059 14 14 14 17 17 17 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 6 6 6 7 7 7 T T T 7 140-159 14 14 14 19 19 19 F F F 8 160-179 13 13 13 16 16 16 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 13 13 13 16 16 16 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 11 11 11 14 14 14 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 14 14 14 19 19 19 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 13 13 13 16 16 16 F F F 23 460-479 13 13 13 16 16 16 T T T 24 480-499 14 14 14 19 19 19 T T T 25 500-519 14 14 14 19 19 19 T T T 26 520-539 14 14 14 19 19 19 F F F 27 540-559 11 11 11 14 14 14 F F F 28 560-579 14 14 14 19 19 19 F F F 29 580-599 14 14 14 19 19 19 F F F Max gen: 14, Max flush: 19 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 15 15 15 20 20 20 F F F 2 040-059 14 14 14 17 17 17 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 15 15 15 20 20 20 F F F 7 140-159 14 14 14 19 19 19 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 13 13 13 16 16 16 F F F 11 220-239 9 9 9 11 11 11 F F F 12 240-259 9 9 9 11 11 11 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 11 11 11 14 14 14 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 7 7 7 9 9 9 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 14 14 14 19 19 19 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 13 13 13 16 16 16 F F F 23 460-479 15 15 15 20 20 20 F F F 24 480-499 15 15 15 20 20 20 F F F 25 500-519 15 15 15 20 20 20 F F F 26 520-539 14 14 14 19 19 19 F F F 27 540-559 11 11 11 14 14 14 F F F 28 560-579 14 14 14 19 19 19 F F F 29 580-599 14 14 14 19 19 19 F F F Max gen: 15, Max flush: 20 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:25.942744424Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:25.996504184Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:25.996934184Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.034419948Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.036168204Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.036222245Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.036240332Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.036258639Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"Crucible 4139d858-89f2-48a1-91e6-22e2f7b98ab7 has session id: 45ab2f31-08f9-444c-9441-41b2b2efbb5b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.037309389Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Upstairs opts: Upstairs UUID: 4139d858-89f2-48a1-91e6-22e2f7b98ab7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.037359011Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Crucible stats registered with UUID: 4139d858-89f2-48a1-91e6-22e2f7b98ab7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.037384292Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:26.037400075Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038245374Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038302742Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"4139d858-89f2-48a1-91e6-22e2f7b98ab7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038331601Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038348686Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038383868Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038401023Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038418458Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038434191Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038820381Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038850172Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038870052Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.038997663Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39497)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.03902538Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59506)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.039271193Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61220)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.039355976Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.039567319Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.039734491Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.040166545Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.040336262Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.040600984Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.040916319Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.041099383Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.041292458Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[0]R flush_numbers[0..12]: [19, 20, 17, 12, 12, 18, 20, 19, 20, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042085Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[0]R generation[0..12]: [14, 15, 14, 10, 10, 14, 15, 14, 15, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042113649Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042132006Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[1]R flush_numbers[0..12]: [19, 20, 17, 12, 12, 18, 20, 19, 20, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042149352Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[1]R generation[0..12]: [14, 15, 14, 10, 10, 14, 15, 14, 15, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042166847Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042191909Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[2]R flush_numbers[0..12]: [19, 20, 17, 12, 12, 18, 20, 19, 20, 1, 16, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042209044Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[2]R generation[0..12]: [14, 15, 14, 10, 10, 14, 15, 14, 15, 1, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042225728Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042242552Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042258425Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042274217Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042289749Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042305511Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"":"downstairs","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042321113Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042335432Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042365023Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"1","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042380905Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"client":"2","":"downstairs","session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042396938Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042411378Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"4139d858-89f2-48a1-91e6-22e2f7b98ab7 is now active with session: 45ab2f31-08f9-444c-9441-41b2b2efbb5b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042426779Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042444185Z","hostname":"bmat-EVT22200007-0000589c","pid":1251,"session_id":"45ab2f31-08f9-444c-9441-41b2b2efbb5b"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042458965Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} {"msg":"Activated sub_volume 4139d858-89f2-48a1-91e6-22e2f7b98ab7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:26.042472743Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 564 len 20480 data: 3 3 4 5 5 02/30 Read block 12 len 28672 03/30 Read block 62 len 20480 04/30 Write block 331 len 8192 data: 2 2 05/30 Write block 328 len 36864 data: 2 2 2 3 3 3 3 3 3 06/30 Read block 91 len 32768 07/30 Flush 08/30 Write block 513 len 8192 data: 3 3 09/30 Read block 34 len 12288 10/30 Write block 408 len 24576 data: 4 4 3 3 4 4 11/30 Write block 537 len 28672 data: 4 4 3 3 3 2 2 12/30 Read block 154 len 36864 13/30 Read block 237 len 4096 14/30 Write block 295 len 36864 data: 3 3 3 4 4 4 4 4 4 15/30 Write block 228 len 4096 data: 2 16/30 Read block 500 len 40960 17/30 Write block 255 len 4096 data: 2 18/30 Flush 19/30 Write block 591 len 28672 data: 3 3 3 3 2 2 2 20/30 Write block 446 len 4096 data: 3 21/30 Write block 217 len 36864 data: 4 3 4 4 4 5 5 5 5 22/30 Flush 23/30 Flush 24/30 Flush 25/30 Write block 576 len 20480 data: 4 3 3 3 3 26/30 Read block 1 len 24576 27/30 Write block 44 len 40960 data: 3 2 3 4 3 3 3 3 4 4 28/30 Read block 74 len 20480 29/30 Read block 519 len 24576 30/30 Write block 136 len 40960 data: 2 3 3 4 5 5 5 6 6 5 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:26.069364192Z","hostname":"bmat-EVT22200007-0000589c","pid":1251} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 15 15 15 20 20 20 F F F 2 040-059 14 14 14 17 17 17 T T T 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 15 15 15 20 20 20 T T T 7 140-159 14 14 14 19 19 19 T T T 8 160-179 15 15 15 20 20 20 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 16 16 16 23 23 23 F F F 11 220-239 16 16 16 23 23 23 F F F 12 240-259 16 16 16 22 22 22 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 16 16 16 22 22 22 F F F 15 300-319 16 16 16 22 22 22 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 16 16 16 23 23 23 F F F 23 460-479 15 15 15 20 20 20 F F F 24 480-499 15 15 15 20 20 20 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 16 16 16 21 21 21 T T T 29 580-599 16 16 16 23 23 23 T T T Max gen: 16, Max flush: 23 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 15 15 15 20 20 20 F F F 2 040-059 17 17 17 24 24 24 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 10 10 10 12 12 12 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 17 17 17 24 24 24 F F F 7 140-159 17 17 17 24 24 24 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 16 16 16 23 23 23 F F F 11 220-239 16 16 16 23 23 23 F F F 12 240-259 16 16 16 22 22 22 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 16 16 16 22 22 22 F F F 15 300-319 16 16 16 22 22 22 F F F 16 320-339 16 16 16 21 21 21 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 16 16 16 23 23 23 F F F 23 460-479 15 15 15 20 20 20 F F F 24 480-499 15 15 15 20 20 20 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 17 17 17 24 24 24 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 17, Max flush: 24 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.425927789Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:30.477645561Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.47811429Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.514924512Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.51557347Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.515634675Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.51567142Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.515700911Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"Crucible efb83de3-7e5b-4160-a2f9-53da8e71b215 has session id: bea6e27a-4895-4929-981e-8feb06fa818a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.516913842Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Upstairs opts: Upstairs UUID: efb83de3-7e5b-4160-a2f9-53da8e71b215, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.516960708Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Crucible stats registered with UUID: efb83de3-7e5b-4160-a2f9-53da8e71b215","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.516981461Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:30.516997083Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.517779263Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518236478Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"efb83de3-7e5b-4160-a2f9-53da8e71b215 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518283615Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518301481Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518333547Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518350382Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518493534Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518528235Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518546513Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518561794Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518578488Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48324)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518595493Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52959)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.51872104Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518741061Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59416)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.518757845Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.519841813Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.519883929Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.519904762Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.520294409Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.520461811Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.520695159Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.520955171Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521175963Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[0]R flush_numbers[0..12]: [19, 20, 24, 12, 12, 18, 24, 24, 20, 1, 23, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521333545Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[0]R generation[0..12]: [14, 15, 17, 10, 10, 14, 17, 17, 15, 1, 16, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521358967Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521376743Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[1]R flush_numbers[0..12]: [19, 20, 24, 12, 12, 18, 24, 24, 20, 1, 23, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521393828Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[1]R generation[0..12]: [14, 15, 17, 10, 10, 14, 17, 17, 15, 1, 16, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521410943Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521428299Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[2]R flush_numbers[0..12]: [19, 20, 24, 12, 12, 18, 24, 24, 20, 1, 23, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521444492Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[2]R generation[0..12]: [14, 15, 17, 10, 10, 14, 17, 17, 15, 1, 16, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521460665Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521477059Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.52149263Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521508012Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521523544Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521538875Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"":"downstairs","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521554988Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521574568Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"0","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521591352Z","hostname":"bmat-EVT22200007-0000589c",Wait for a query_work_queue command to finish before sending IO "pid":1259,"client":"1","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521622085Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"client":"2","":"downstairs","session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521638288Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521652678Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"efb83de3-7e5b-4160-a2f9-53da8e71b215 is now active with session: bea6e27a-4895-4929-981e-8feb06fa818a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.52166837Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521682529Z","hostname":"bmat-EVT22200007-0000589c","pid":1259,"session_id":"bea6e27a-4895-4929-981e-8feb06fa818a"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521696999Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} {"msg":"Activated sub_volume efb83de3-7e5b-4160-a2f9-53da8e71b215","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:30.521710586Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 8192 data: 3 4 02/30 Write block 242 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 222 len 12288 data: 6 6 6 04/30 Write block 327 len 8192 data: 2 3 05/30 Write block 90 len 8192 data: 2 2 06/30 Read block 567 len 20480 07/30 Read block 181 len 40960 08/30 Write block 322 len 4096 data: 3 09/30 Read block 343 len 24576 10/30 Read block 16 len 24576 11/30 Read block 299 len 16384 12/30 Read block 476 len 4096 13/30 Read block 467 len 12288 14/30 Read block 541 len 16384 15/30 Write block 567 len 16384 data: 6 6 3 4 16/30 Read block 498 len 36864 17/30 Read block 12 len 32768 18/30 Write block 89 len 36864 data: 2 3 3 2 2 2 2 2 2 19/30 Flush 20/30 Write block 486 len 4096 data: 3 21/30 Read block 299 len 4096 22/30 Flush 23/30 Flush 24/30 Write block 22 len 4096 data: 3 25/30 Read block 140 len 12288 26/30 Read block 432 len 36864 27/30 Write block 314 len 20480 data: 3 2 3 3 3 28/30 Write block 448 len 12288 data: 2 2 2 29/30 Write block 161 len 12288 data: 3 3 2 30/30 Read block 26 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 7 7 4 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:30.545396066Z","hostname":"bmat-EVT22200007-0000589c","pid":1259} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 15 15 15 20 20 20 T T T 2 040-059 17 17 17 24 24 24 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 18 18 18 25 25 25 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 17 17 17 24 24 24 F F F 7 140-159 17 17 17 24 24 24 F F F 8 160-179 15 15 15 20 20 20 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 16 16 16 22 22 22 F F F 15 300-319 16 16 16 22 22 22 T T T 16 320-339 18 18 18 25 25 25 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 16 16 16 23 23 23 T T T 23 460-479 15 15 15 20 20 20 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 18 18 18 25 25 25 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 18, Max flush: 26 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 19 19 19 F F F 1 020-039 19 19 19 27 27 27 F F F 2 040-059 17 17 17 24 24 24 F F F 3 060-079 10 10 10 12 12 12 F F F 4 080-099 18 18 18 25 25 25 F F F 5 100-119 14 14 14 18 18 18 F F F 6 120-139 17 17 17 24 24 24 F F F 7 140-159 17 17 17 24 24 24 F F F 8 160-179 19 19 19 27 27 27 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 12 12 12 15 15 15 F F F 14 280-299 16 16 16 22 22 22 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 18 18 18 25 25 25 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 F F F 20 400-419 16 16 16 22 22 22 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 19 19 19 27 27 27 F F F 23 460-479 15 15 15 20 20 20 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 18 18 18 25 25 25 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 19, Max flush: 27 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.889866933Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:34.941390287Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.941824575Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.978122158Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.978820046Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.978875951Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.978903307Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.978932096Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"Crucible 60b6b774-2ac2-4377-b655-6959e71453b6 has session id: 53034649-53ed-48b7-af3c-2c71b9bf85df","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.979726992Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Upstairs opts: Upstairs UUID: 60b6b774-2ac2-4377-b655-6959e71453b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.979840023Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Crucible stats registered with UUID: 60b6b774-2ac2-4377-b655-6959e71453b6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.979863872Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:34.979879414Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980647325Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980726998Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"60b6b774-2ac2-4377-b655-6959e71453b6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980745205Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980763132Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980799186Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.98081611Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980834798Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980853617Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980871934Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980888228Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.980905613Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.981037623Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"ds_connection connected from Ok(127.0.0.1:54811)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.981070751Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"ds_connection connected from Ok(127.0.0.1:44221)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.981091904Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"ds_connection connected from Ok(127.0.0.1:42382)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.9812753Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.982223109Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.982286208Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.982941719Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.983430659Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.98352392Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.983913437Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984128848Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984408169Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[0]R flush_numbers[0..12]: [19, 27, 24, 12, 25, 18, 24, 24, 27, 1, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984738666Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[0]R generation[0..12]: [14, 19, 17, 10, 18, 14, 17, 17, 19, 1, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984780581Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984812867Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[1]R flush_numbers[0..12]: [19, 27, 24, 12, 25, 18, 24, 24, 27, 1, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984843129Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[1]R generation[0..12]: [14, 19, 17, 10, 18, 14, 17, 17, 19, 1, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984871778Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984900497Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[2]R flush_numbers[0..12]: [19, 27, 24, 12, 25, 18, 24, 24, 27, 1, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984928083Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[2]R generation[0..12]: [14, 19, 17, 10, 18, 14, 17, 17, 19, 1, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.98495603Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.984997094Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985027456Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985044622Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985060624Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.9850975Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"":"downstairs","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985115617Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985130457Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"0","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985162633Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"1","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985179037Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"client":"2","":"downstairs","session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.98519536Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.98521026Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"60b6b774-2ac2-4377-b655-6959e71453b6 is now active with session: 53034649-53ed-48b7-af3c-2c71b9bf85df","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985226123Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985240913Z","hostname":"bmat-EVT22200007-0000589c","pid":1267,"session_id":"53034649-53ed-48b7-af3c-2c71b9bf85df"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985255834Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} {"msg":"Activated sub_volume 60b6b774-2ac2-4377-b655-6959e71453b6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:34.985269953Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 561 len 8192 data: 3 3 02/30 Read block 344 len 24576 03/30 Write block 463 len 28672 data: 6 6 6 5 5 5 6 04/30 Read block 108 len 4096 05/30 Flush 06/30 Write block 190 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Write block 2 len 24576 data: 2 2 2 2 2 2 08/30 Flush 09/30 Write block 327 len 24576 data: 3 4 3 3 4 4 10/30 Read block 75 len 40960 11/30 Write block 281 len 4096 data: 4 12/30 Read block 216 len 36864 13/30 Flush 14/30 Write block 387 len 36864 data: 4 4 4 4 5 5 4 4 3 15/30 Read block 266 len 36864 16/30 Read block 76 len 16384 17/30 Read block 165 len 16384 18/30 Write block 190 len 24576 data: 3 3 3 3 3 3 19/30 Write block 115 len 4096 data: 3 20/30 Read block 152 len 24576 21/30 Write block 267 len 20480 data: 3 3 3 3 3 22/30 Write block 398 len 12288 data: 2 2 4 23/30 Read block 213 len 24576 24/30 Read block 69 len 32768 25/30 Read block 8 len 12288 26/30 Read block 312 len 8192 27/30 Write block 423 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 227 len 24576 29/30 Write block 78 len 12288 data: 4 4 4 30/30 Write block 462 len 16384 data: 5 7 7 7 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:35.011589501Z","hostname":"bmat-EVT22200007-0000589c","pid":1267} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 19 19 19 27 27 27 F F F 2 040-059 17 17 17 24 24 24 F F F 3 060-079 10 10 10 12 12 12 T T T 4 080-099 18 18 18 25 25 25 T T T 5 100-119 14 14 14 18 18 18 T T T 6 120-139 17 17 17 24 24 24 F F F 7 140-159 17 17 17 24 24 24 F F F 8 160-179 19 19 19 27 27 27 F F F 9 180-199 20 20 20 29 29 29 T T T 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 12 12 12 15 15 15 T T T 14 280-299 20 20 20 30 30 30 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 12 12 12 15 15 15 T T T 20 400-419 16 16 16 22 22 22 T T T 21 420-439 13 13 13 16 16 16 T T T 22 440-459 19 19 19 27 27 27 F F F 23 460-479 20 20 20 28 28 28 T T T 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 20 20 20 28 28 28 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 20, Max flush: 30 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 19 19 19 27 27 27 F F F 2 040-059 17 17 17 24 24 24 F F F 3 060-079 21 21 21 31 31 31 F F F 4 080-099 21 21 21 31 31 31 F F F 5 100-119 21 21 21 31 31 31 F F F 6 120-139 17 17 17 24 24 24 F F F 7 140-159 17 17 17 24 24 24 F F F 8 160-179 19 19 19 27 27 27 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 21 21 21 31 31 31 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 21 21 21 31 31 31 F F F 20 400-419 21 21 21 31 31 31 F F F 21 420-439 21 21 21 31 31 31 F F F 22 440-459 19 19 19 27 27 27 F F F 23 460-479 21 21 21 31 31 31 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 20 20 20 28 28 28 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 21, Max flush: 31 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.39829109Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.451237135Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.451932768Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.451990256Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.452012291Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.45203136Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"Crucible e9f77ec0-c802-4573-b72a-c7b2abc4328d has session id: 4f709fa5-6969-4812-a13d-5f947bdd3700","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453006124Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Upstairs opts: Upstairs UUID: e9f77ec0-c802-4573-b72a-c7b2abc4328d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453056497Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Crucible stats registered with UUID: e9f77ec0-c802-4573-b72a-c7b2abc4328d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453075256Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:39.453090477Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453806782Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453834158Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"e9f77ec0-c802-4573-b72a-c7b2abc4328d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453963012Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.453984355Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454016551Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454033376Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454050661Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454066624Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454083759Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454099742Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454121546Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454192902Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"ds_connection connected from Ok(127.0.0.1:53571)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454614876Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"ds_connection connected from Ok(127.0.0.1:51924)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454676802Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"ds_connection connected from Ok(127.0.0.1:34414)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.454697224Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.45568728Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.4557561Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.456153624Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.456500754Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.456530605Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.457159842Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.457432721Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.457493385Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[0]R flush_numbers[0..12]: [29, 27, 24, 31, 31, 31, 24, 24, 27, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.457961262Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[0]R generation[0..12]: [20, 19, 17, 21, 21, 21, 17, 17, 19, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.457995662Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458022457Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[1]R flush_numbers[0..12]: [29, 27, 24, 31, 31, 31, 24, 24, 27, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458041757Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[1]R generation[0..12]: [20, 19, 17, 21, 21, 21, 17, 17, 19, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458058341Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458075796Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[2]R flush_numbers[0..12]: [29, 27, 24, 31, 31, 31, 24, 24, 27, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.45809245Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[2]R generation[0..12]: [20, 19, 17, 21, 21, 21, 17, 17, 19, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458108854Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458125308Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458140649Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.45815594Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458170991Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458186272Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"":"downstairs","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458202265Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458216725Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"0","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458233228Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"1","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458249382Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"client":"2","":"downstairs","session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458268831Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458283972Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"e9f77ec0-c802-4573-b72a-c7b2abc4328d is now active with session: 4f709fa5-6969-4812-a13d-5f947bdd3700","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458300526Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458315226Z","hostname":"bmat-EVT22200007-0000589c","pid":1275,"session_id":"4f709fa5-6969-4812-a13d-5f947bdd3700"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458330357Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} {"msg":"Activated sub_volume e9f77ec0-c802-4573-b72a-c7b2abc4328d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:39.458344296Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 28672 data: 2 3 4 3 4 4 5 02/30 Write block 164 len 36864 data: 2 2 2 4 4 4 4 4 4 03/30 Write block 34 len 40960 data: 3 3 3 3 2 2 3 3 4 4 04/30 Read block 147 len 24576 05/30 Read block 140 len 36864 06/30 Flush 07/30 Write block 296 len 12288 data: 4 4 5 08/30 Read block 125 len 24576 09/30 Read block 492 len 20480 10/30 Write block 419 len 16384 data: 4 4 2 2 11/30 Flush 12/30 Write block 145 len 16384 data: 6 5 2 2 13/30 Write block 391 len 40960 data: 6 6 5 5 4 3 2 3 3 5 14/30 Write block 272 len 36864 data: 3 3 3 3 3 3 4 4 4 15/30 Write block 433 len 40960 data: 5 5 5 6 5 5 3 3 3 4 16/30 Flush 17/30 Flush 18/30 Write block 387 len 40960 data: 5 5 5 5 7 7 6 6 5 4 19/30 Read block 453 len 36864 20/30 Read block 147 len 32768 21/30 Read block 238 len 40960 22/30 Read block 203 len 32768 23/30 Write block 112 len 8192 data: 3 3 24/30 Read block 595 len 12288 25/30 Read block 63 len 20480 26/30 Read block 100 len 4096 27/30 Write block 176 len 16384 data: 3 2 3 3 28/30 Write block 172 len 4096 data: 5 29/30 Write block 57 len 20480 data: 2 2 2 2 2 30/30 Read block 232 len 16384 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 13 12 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:39.486136868Z","hostname":"bmat-EVT22200007-0000589c","pid":1275} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 22 22 22 32 32 32 T T T 3 060-079 21 21 21 31 31 31 T T T 4 080-099 22 22 22 32 32 32 F F F 5 100-119 22 22 22 32 32 32 T T T 6 120-139 17 17 17 24 24 24 F F F 7 140-159 22 22 22 34 34 34 F F F 8 160-179 22 22 22 32 32 32 T T T 9 180-199 21 21 21 31 31 31 F F F 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 22 22 22 34 34 34 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 22 22 22 34 34 34 T T T 20 400-419 22 22 22 34 34 34 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 21 21 21 31 31 31 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 20 20 20 28 28 28 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 22, Max flush: 34 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 23 23 23 35 35 35 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 17 17 17 24 24 24 F F F 7 140-159 22 22 22 34 34 34 F F F 8 160-179 23 23 23 35 35 35 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 18 18 18 25 25 25 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 22 22 22 34 34 34 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 14 14 14 19 19 19 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 22 22 22 34 34 34 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 21 21 21 31 31 31 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 16 16 16 22 22 22 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 16 16 16 22 22 22 F F F 28 560-579 20 20 20 28 28 28 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 23, Max flush: 35 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.799722453Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:43.851048394Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.851425886Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.88768969Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.888513064Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.88865278Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.888685958Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.88871634Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"Crucible a6cd2d6b-eb53-4864-9591-27dcd6fa2df0 has session id: dd7d8e21-c161-4e7d-82a8-efe474319890","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.889704221Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Upstairs opts: Upstairs UUID: a6cd2d6b-eb53-4864-9591-27dcd6fa2df0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.889763162Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Crucible stats registered with UUID: a6cd2d6b-eb53-4864-9591-27dcd6fa2df0","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.889787291Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:43.889807954Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.890557287Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.890592589Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"a6cd2d6b-eb53-4864-9591-27dcd6fa2df0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.89072517Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.890758318Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891082351Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891501338Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891676988Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891699063Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891719765Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891738303Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891757713Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891777072Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"ds_connection connected from Ok(127.0.0.1:62953)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891796492Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"ds_connection connected from Ok(127.0.0.1:57142)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891815912Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"ds_connection connected from Ok(127.0.0.1:41776)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.891835181Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.892534783Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.892603814Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.893253213Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.893597106Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.893814822Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894192464Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894253138Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894386962Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[0]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 35, 24, 34, 35, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.89485496Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[0]R generation[0..12]: [20, 22, 23, 23, 22, 23, 17, 22, 23, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894880342Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894898449Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[1]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 35, 24, 34, 35, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894915383Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[1]R generation[0..12]: [20, 22, 23, 23, 22, 23, 17, 22, 23, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894932158Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894949182Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[2]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 35, 24, 34, 35, 31, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894965416Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[2]R generation[0..12]: [20, 22, 23, 23, 22, 23, 17, 22, 23, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894981529Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.894998273Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895022182Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895038605Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895054117Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895069398Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"":"downstairs","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.89508493Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.89509945Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"0","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895115443Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"1","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895130944Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"client":"2","":"downstairs","session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895146686Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895161086Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"a6cd2d6b-eb53-4864-9591-27dcd6fa2df0 is now active with session: dd7d8e21-c161-4e7d-82a8-efe474319890","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895176598Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895190877Z","hostname":"bmat-EVT22200007-0000589c","pid":1283,"session_id":"dd7d8e21-c161-4e7d-82a8-efe474319890"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895205367Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} {"msg":"Activated sub_volume a6cd2d6b-eb53-4864-9591-27dcd6fa2df0","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:43.895326053Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 24576 data: 5 3 3 3 3 3 02/30 Write block 352 len 28672 data: 2 2 2 2 2 3 3 03/30 Read block 487 len 32768 04/30 Flush 05/30 Write block 262 len 20480 data: 2 2 2 3 3 06/30 Write block 116 len 28672 data: 3 3 2 2 2 2 2 07/30 Read block 534 len 36864 08/30 Write block 216 len 8192 data: 3 5 09/30 Read block 102 len 20480 10/30 Write block 365 len 16384 data: 4 4 3 3 11/30 Flush 12/30 Write block 173 len 4096 data: 4 13/30 Flush 14/30 Write block 570 len 28672 data: 5 4 4 4 4 4 5 15/30 Read block 148 len 8192 16/30 Read block 481 len 32768 17/30 Read block 587 len 36864 18/30 Read block 340 len 12288 19/30 Write block 511 len 32768 data: 2 2 4 4 3 3 3 3 20/30 Write block 159 len 40960 data: 3 3 4 4 3 3 3 3 5 5 21/30 Write block 432 len 16384 data: 5 6 6 6 22/30 Write block 340 len 4096 data: 3 23/30 Write block 554 len 24576 data: 2 2 4 5 4 4 24/30 Flush 25/30 Write block 544 len 4096 data: 3 26/30 Read block 132 len 40960 27/30 Read block 369 len 16384 28/30 Read block 424 len 40960 29/30 Read block 7 len 40960 30/30 Read block 518 len 40960 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:43.921422606Z","hostname":"bmat-EVT22200007-0000589c","pid":1283} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 23 23 23 35 35 35 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 24 24 24 37 37 37 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 24 24 24 37 37 37 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 21 21 21 31 31 31 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 24 24 24 39 39 39 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 24 24 24 39 39 39 T T T 28 560-579 24 24 24 39 39 39 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 24, Max flush: 39 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 23 23 23 35 35 35 F F F 4 080-099 22 22 22 32 32 32 F F F 5 100-119 24 24 24 37 37 37 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 24 24 24 37 37 37 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 21 21 21 31 31 31 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 24 24 24 39 39 39 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 25 25 25 40 40 40 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 17 17 17 24 24 24 F F F Max gen: 25, Max flush: 40 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.211373507Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.264775144Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.265469164Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.265525099Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.265544709Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.265562736Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"Crucible 7f5cca2d-e495-4fae-94b9-b3ef29c06922 has session id: e897850e-1815-472c-b284-c922fda930fc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.266680152Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Upstairs opts: Upstairs UUID: 7f5cca2d-e495-4fae-94b9-b3ef29c06922, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.266730976Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Crucible stats registered with UUID: 7f5cca2d-e495-4fae-94b9-b3ef29c06922","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.266757971Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:48.266774194Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.267464126Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.26749568Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"7f5cca2d-e495-4fae-94b9-b3ef29c06922 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.267623332Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.267644405Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.267674566Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.267691361Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268183518Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268389479Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268411043Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268427377Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268444562Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268461186Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43658)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268477679Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54233)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268494233Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35209)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.268519625Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.26905953Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.26978808Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.269906894Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.270250547Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.270973396Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.271060374Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.271098041Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.271939683Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[0]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.27277927Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[0]R generation[0..12]: [20, 22, 23, 23, 22, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272805734Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272823441Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[1]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272840486Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[1]R generation[0..12]: [20, 22, 23, 23, 22, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272856959Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272880848Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[2]R flush_numbers[0..12]: [29, 32, 35, 35, 32, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272898194Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[2]R generation[0..12]: [20, 22, 23, 23, 22, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272914527Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272931261Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272946893Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272962635Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.272978127Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273028961Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"":"downstairs","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273046567Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273061197Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"0","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273077561Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"1","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273093443Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"client":"2","":"downstairs","session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273109366Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273123765Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"7f5cca2d-e495-4fae-94b9-b3ef29c06922 is now active with session: e897850e-1815-472c-b284-c922fda930fc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273139297Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273157544Z","hostname":"bmat-EVT22200007-0000589c","pid":1291,"session_id":"e897850e-1815-472c-b284-c922fda930fc"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273172355Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} {"msg":"Activated sub_volume 7f5cca2d-e495-4fae-94b9-b3ef29c06922","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:48.273185992Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 4096 data: 4 02/30 Read block 597 len 4096 03/30 Read block 19 len 24576 04/30 Read block 502 len 12288 05/30 Read block 292 len 28672 06/30 Read block 236 len 8192 07/30 Read block 303 len 8192 08/30 Flush 09/30 Write block 481 len 32768 data: 4 3 3 3 3 4 3 2 10/30 Write block 57 len 24576 data: 3 3 3 3 3 2 11/30 Flush 12/30 Write block 385 len 24576 data: 3 3 6 6 6 6 13/30 Read block 500 len 24576 14/30 Read block 329 len 12288 15/30 Write block 468 len 20480 data: 6 7 6 4 3 16/30 Read block 198 len 36864 17/30 Flush 18/30 Flush 19/30 Write block 508 len 40960 data: 3 2 2 3 3 5 5 4 4 4 20/30 Write block 78 len 40960 data: 5 5 5 3 3 2 2 2 2 2 21/30 Read block 229 len 12288 22/30 Write block 499 len 12288 data: 3 4 4 23/30 Write block 589 len 12288 data: 3 3 4 24/30 Flush 25/30 Write block 78 len 20480 data: 6 6 6 4 4 26/30 Read block 368 len 8192 27/30 Read block 221 len 8192 28/30 Read block 260 len 12288 29/30 Write block 488 len 4096 data: 3 30/30 Read block 334 len 16384 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:48.299414605Z","hostname":"bmat-EVT22200007-0000589c","pid":1291} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 26 26 26 42 42 42 F F F 3 060-079 26 26 26 45 45 45 T T T 4 080-099 26 26 26 45 45 45 T T T 5 100-119 24 24 24 37 37 37 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 24 24 24 37 37 37 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 26 26 26 45 45 45 T T T 25 500-519 26 26 26 45 45 45 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 26 26 26 45 45 45 F F F Max gen: 26, Max flush: 45 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 F F F 2 040-059 26 26 26 42 42 42 F F F 3 060-079 27 27 27 46 46 46 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 24 24 24 37 37 37 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 31 31 31 F F F 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 18 18 18 25 25 25 F F F 13 260-279 24 24 24 37 37 37 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 20 20 20 30 30 30 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 26 26 26 43 43 43 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 22 22 22 34 34 34 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 27 27 27 46 46 46 F F F 25 500-519 26 26 26 45 45 45 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 26 26 26 45 45 45 F F F Max gen: 27, Max flush: 46 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.607969138Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.661673405Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.662735768Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.662791793Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.662813257Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.662838679Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"Crucible 4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae has session id: 1ee294db-ab8c-4a76-b46a-d5243015492b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.663838595Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Upstairs opts: Upstairs UUID: 4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.663978491Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Crucible stats registered with UUID: 4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664030748Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:52.664047251Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664646327Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664677381Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664845865Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664899255Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664931551Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664949267Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664967073Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.664983206Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665000291Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665016434Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665040764Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665057398Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48917)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665213688Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52993)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.66560563Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42445)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665670793Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.665970486Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.666170055Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.666616047Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.666774161Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.66686698Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.667431125Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.667465104Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.667560199Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[0]R flush_numbers[0..12]: [29, 32, 42, 46, 46, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668302006Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[0]R generation[0..12]: [20, 22, 26, 27, 27, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668326366Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668350225Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[1]R flush_numbers[0..12]: [29, 32, 42, 46, 46, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.66836712Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[1]R generation[0..12]: [20, 22, 26, 27, 27, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668383503Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668400598Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[2]R flush_numbers[0..12]: [29, 32, 42, 46, 46, 37, 37, 39, 39, 31, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668416811Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[2]R generation[0..12]: [20, 22, 26, 27, 27, 24, 24, 24, 24, 21, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668432724Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668449388Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668464619Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.66847972Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668494931Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668509912Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"":"downstairs","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668525223Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668539172Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"0","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668554653Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"1","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668573221Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"client":"2","":"downstairs","session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668589064Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668603543Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae is now active with session: 1ee294db-ab8c-4a76-b46a-d5243015492b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668618724Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668632673Z","hostname":"bmat-EVT22200007-0000589c","pid":1299,"session_id":"1ee294db-ab8c-4a76-b46a-d5243015492b"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668646862Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} {"msg":"Activated sub_volume 4c83cfc0-fe4a-4e42-97d4-054e5b0c40ae","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:52.668659949Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 485 len 16384 data: 4 5 4 4 03/30 Read block 576 len 32768 04/30 Read block 463 len 4096 05/30 Read block 452 len 36864 06/30 Write block 394 len 4096 data: 7 07/30 Write block 339 len 4096 data: 3 08/30 Read block 517 len 28672 09/30 Read block 16 len 36864 10/30 Flush 11/30 Write block 92 len 36864 data: 3 3 3 3 3 3 3 4 5 12/30 Read block 81 len 28672 13/30 Write block 192 len 16384 data: 4 4 4 4 14/30 Write block 144 len 12288 data: 7 7 6 15/30 Write block 250 len 24576 data: 2 3 3 2 2 3 16/30 Read block 295 len 16384 17/30 Flush 18/30 Write block 264 len 12288 data: 3 4 4 19/30 Read block 476 len 16384 20/30 Read block 409 len 28672 21/30 Read block 72 len 40960 22/30 Write block 443 len 24576 data: 3 3 3 4 3 3 23/30 Write block 39 len 8192 data: 3 4 24/30 Read block 193 len 8192 25/30 Read block 489 len 28672 26/30 Read block 478 len 12288 27/30 Write block 456 len 40960 data: 3 3 3 3 4 5 6 8 8 8 28/30 Read block 224 len 28672 29/30 Write block 178 len 16384 data: 4 4 2 2 30/30 Read block 536 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 12 13 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:52.696074247Z","hostname":"bmat-EVT22200007-0000589c","pid":1299} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 22 22 22 32 32 32 T T T 2 040-059 26 26 26 42 42 42 T T T 3 060-079 27 27 27 46 46 46 F F F 4 080-099 28 28 28 49 49 49 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 28 28 28 49 49 49 F F F 8 160-179 24 24 24 39 39 39 T T T 9 180-199 28 28 28 49 49 49 T T T 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 28 28 28 49 49 49 F F F 13 260-279 24 24 24 37 37 37 T T T 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 28 28 28 48 48 48 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 22 22 22 34 34 34 T T T 23 460-479 26 26 26 43 43 43 T T T 24 480-499 28 28 28 48 48 48 F F F 25 500-519 26 26 26 45 45 45 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 26 26 26 45 45 45 F F F Max gen: 28, Max flush: 49 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 29 29 29 50 50 50 F F F 2 040-059 29 29 29 50 50 50 F F F 3 060-079 27 27 27 46 46 46 F F F 4 080-099 28 28 28 49 49 49 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 28 28 28 49 49 49 F F F 8 160-179 29 29 29 50 50 50 F F F 9 180-199 29 29 29 50 50 50 F F F 10 200-219 24 24 24 37 37 37 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 28 28 28 49 49 49 F F F 13 260-279 29 29 29 50 50 50 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 19 19 19 27 27 27 F F F 16 320-339 28 28 28 48 48 48 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 37 37 37 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 29 29 29 50 50 50 F F F 23 460-479 29 29 29 50 50 50 F F F 24 480-499 28 28 28 48 48 48 F F F 25 500-519 26 26 26 45 45 45 F F F 26 520-539 16 16 16 22 22 22 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 26 26 26 45 45 45 F F F Max gen: 29, Max flush: 50 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.028919017Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.083180203Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.084192033Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.084508501Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.084735525Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.084999475Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"Crucible ceda15b8-c789-4661-9ac7-3e1ae760ddd9 has session id: af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.086346892Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Upstairs opts: Upstairs UUID: ceda15b8-c789-4661-9ac7-3e1ae760ddd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.086625051Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Crucible stats registered with UUID: ceda15b8-c789-4661-9ac7-3e1ae760ddd9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.086855903Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:57.087179706Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088299086Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088596585Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"ceda15b8-c789-4661-9ac7-3e1ae760ddd9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088651868Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088783748Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088822888Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088848761Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088867489Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088884293Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088902601Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.088919335Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.089009019Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.089033639Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"ds_connection connected from Ok(127.0.0.1:47825)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.089134966Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"ds_connection connected from Ok(127.0.0.1:55370)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.089490043Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"ds_connection connected from Ok(127.0.0.1:53518)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.089644569Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.09017314Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.090212671Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.090358249Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.090897332Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.090929919Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091090307Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091718412Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091743112Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[0]R flush_numbers[0..12]: [29, 50, 50, 46, 49, 49, 37, 49, 50, 50, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091854751Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[0]R generation[0..12]: [20, 29, 29, 27, 28, 28, 24, 28, 29, 29, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091877698Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091895284Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[1]R flush_numbers[0..12]: [29, 50, 50, 46, 49, 49, 37, 49, 50, 50, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.09191275Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[1]R generation[0..12]: [20, 29, 29, 27, 28, 28, 24, 28, 29, 29, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091929404Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091946829Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[2]R flush_numbers[0..12]: [29, 50, 50, 46, 49, 49, 37, 49, 50, 50, 37, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091963914Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[2]R generation[0..12]: [20, 29, 29, 27, 28, 28, 24, 28, 29, 29, 24, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091980628Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.091997824Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092013456Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092029559Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092048978Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092064931Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"":"downstairs","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092081064Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092095904Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"0","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092112498Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"1","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092128561Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"client":"2","":"downstairs","session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092144694Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092159404Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"ceda15b8-c789-4661-9ac7-3e1ae760ddd9 is now active with session: af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092175116Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092189636Z","hostname":"bmat-EVT22200007-0000589c","pid":1307,"session_id":"af0de2aa-f4a1-4f22-9d6e-ec2e7ca19396"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092204256Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} {"msg":"Activated sub_volume ceda15b8-c789-4661-9ac7-3e1ae760ddd9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:03:57.092228215Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 538 len 36864 data: 5 4 4 4 3 3 5 2 2 02/30 Write block 163 len 8192 data: 4 4 03/30 Flush 04/30 Write block 58 len 12288 data: 4 4 4 05/30 Write block 334 len 4096 data: 4 06/30 Read block 393 len 40960 07/30 Read block 158 len 40960 08/30 Read block 153 len 36864 09/30 Write block 476 len 32768 data: 2 3 4 4 4 5 4 4 10/30 Write block 261 len 12288 data: 2 3 3 11/30 Read block 226 len 8192 12/30 Write block 315 len 24576 data: 3 4 4 4 3 3 13/30 Write block 513 len 28672 data: 6 6 5 5 5 4 2 14/30 Read block 553 len 32768 15/30 Read block 84 len 20480 16/30 Write block 373 len 16384 data: 6 6 5 5 17/30 Flush 18/30 Write block 363 len 16384 data: 3 4 5 5 19/30 Write block 26 len 36864 data: 5 5 4 3 3 3 3 2 4 20/30 Write block 93 len 28672 data: 4 4 4 4 4 4 5 21/30 Flush 22/30 Write block 206 len 4096 data: 3 23/30 Read block 1 len 36864 24/30 Read block 449 len 4096 25/30 Write block 266 len 24576 data: 5 4 4 4 4 4 26/30 Write block 594 len 4096 data: 4 27/30 Read block 199 len 40960 28/30 Write block 160 len 8192 data: 4 5 29/30 Read block 422 len 24576 30/30 Write block 505 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:03:57.117438394Z","hostname":"bmat-EVT22200007-0000589c","pid":1307} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 30 30 30 52 52 52 F F F 4 080-099 30 30 30 53 53 53 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 28 28 28 49 49 49 F F F 8 160-179 30 30 30 51 51 51 T T T 9 180-199 29 29 29 50 50 50 F F F 10 200-219 24 24 24 37 37 37 T T T 11 220-239 18 18 18 25 25 25 F F F 12 240-259 28 28 28 49 49 49 F F F 13 260-279 30 30 30 52 52 52 T T T 14 280-299 22 22 22 34 34 34 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 30 30 30 53 53 53 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 29 29 29 50 50 50 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 30 30 30 52 52 52 T T T 26 520-539 30 30 30 51 51 51 F F F 27 540-559 30 30 30 51 51 51 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 26 26 26 45 45 45 T T T Max gen: 30, Max flush: 53 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 30 30 30 52 52 52 F F F 3 060-079 30 30 30 52 52 52 F F F 4 080-099 30 30 30 53 53 53 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 28 28 28 49 49 49 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 29 29 29 50 50 50 F F F 10 200-219 31 31 31 54 54 54 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 28 28 28 49 49 49 F F F 13 260-279 31 31 31 54 54 54 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 30 30 30 53 53 53 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 22 22 22 34 34 34 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 29 29 29 50 50 50 F F F 23 460-479 30 30 30 52 52 52 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 31 31 31 54 54 54 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 30 30 30 51 51 51 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 31, Max flush: 54 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.44226Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.495828678Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.496572841Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.49662129Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.496639628Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.496657043Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"Crucible d878fcb5-2427-4566-8c2e-f111d15f676a has session id: 0a010365-fbc1-4af6-8247-83bddf9ca5ff","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.49755986Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Upstairs opts: Upstairs UUID: d878fcb5-2427-4566-8c2e-f111d15f676a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.497739448Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Crucible stats registered with UUID: d878fcb5-2427-4566-8c2e-f111d15f676a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.497761403Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:01.497785322Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.49851281Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498539906Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"d878fcb5-2427-4566-8c2e-f111d15f676a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498684722Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498704903Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498735025Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.49875211Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498769485Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498785899Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498803214Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498819137Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498836653Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.498919192Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:55188)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.4990653Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:52245)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.499087175Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:01.499374343Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:01.499415026Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:01.499435257Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.499619995Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.50042373Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.500455435Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.501448096Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.50164573Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.502112785Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:01.502316552Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.500512836Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:41523)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.500613412Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.502025892Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.502868205Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[0]R flush_numbers[0..12]: [29, 53, 52, 52, 53, 49, 37, 49, 54, 50, 54, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503747694Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[0]R generation[0..12]: [20, 30, 30, 30, 30, 28, 24, 28, 31, 29, 31, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503781584Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503800562Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[1]R flush_numbers[0..12]: [29, 53, 52, 52, 53, 49, 37, 49, 54, 50, 54, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503817627Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[1]R generation[0..12]: [20, 30, 30, 30, 30, 28, 24, 28, 31, 29, 31, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.50383377Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503851006Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[2]R flush_numbers[0..12]: [29, 53, 52, 52, 53, 49, 37, 49, 54, 50, 54, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503867199Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[2]R generation[0..12]: [20, 30, 30, 30, 30, 28, 24, 28, 31, 29, 31, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.503979499Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504041325Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504060013Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504076186Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.50409239Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504109484Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"":"downstairs","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504125487Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504139877Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"0","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.5041562Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"1","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504182283Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"client":"2","":"downstairs","session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504199038Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504213908Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"d878fcb5-2427-4566-8c2e-f111d15f676a is now active with session: 0a010365-fbc1-4af6-8247-83bddf9ca5ff","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504229169Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504243338Z","hostname":"bmat-EVT22200007-0000589c","pid":1315,"session_id":"0a010365-fbc1-4af6-8247-83bddf9ca5ff"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504258018Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} {"msg":"Activated sub_volume d878fcb5-2427-4566-8c2e-f111d15f676a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:11.504271977Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 32768 data: 5 5 4 4 5 5 4 4 02/30 Read block 87 len 4096 03/30 Flush 04/30 Write block 466 len 24576 data: 6 6 7 8 7 5 05/30 Read block 598 len 8192 06/30 Read block 327 len 4096 07/30 Read block 368 len 20480 08/30 Read block 533 len 4096 09/30 Read block 596 len 16384 10/30 Write block 84 len 40960 data: 3 3 3 3 2 3 4 4 4 5 11/30 Write block 250 len 36864 data: 3 4 4 3 3 4 3 3 3 12/30 Read block 130 len 24576 13/30 Read block 368 len 32768 14/30 Flush 15/30 Write block 371 len 28672 data: 5 4 7 7 6 6 3 16/30 Write block 366 len 4096 data: 6 17/30 Flush 18/30 Write block 63 len 4096 data: 2 19/30 Read block 419 len 36864 20/30 Read block 90 len 36864 21/30 Write block 159 len 24576 data: 4 5 6 5 5 5 22/30 Write block 448 len 24576 data: 4 3 3 2 2 2 23/30 Read block 150 len 20480 24/30 Flush 25/30 Write block 196 len 40960 data: 3 3 3 2 2 2 2 3 2 2 26/30 Flush 27/30 Write block 514 len 20480 data: 7 6 6 6 5 28/30 Write block 48 len 40960 data: 4 4 4 4 5 5 4 3 3 4 29/30 Write block 266 len 12288 data: 6 5 5 30/30 Read block 181 len 40960 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 4 4 1 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:11.533803066Z","hostname":"bmat-EVT22200007-0000589c","pid":1315} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 30 30 30 52 52 52 T T T 3 060-079 32 32 32 58 58 58 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 32 32 32 58 58 58 F F F 8 160-179 32 32 32 58 58 58 F F F 9 180-199 32 32 32 59 59 59 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 32 32 32 56 56 56 F F F 13 260-279 31 31 31 54 54 54 T T T 14 280-299 22 22 22 34 34 34 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 32 32 32 57 57 57 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 32 32 32 55 55 55 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 31 31 31 54 54 54 T T T 26 520-539 30 30 30 51 51 51 F F F 27 540-559 30 30 30 51 51 51 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 32, Max flush: 59 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 33 33 33 60 60 60 F F F 3 060-079 32 32 32 58 58 58 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 32 32 32 58 58 58 F F F 8 160-179 32 32 32 58 58 58 F F F 9 180-199 32 32 32 59 59 59 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 18 18 18 25 25 25 F F F 12 240-259 32 32 32 56 56 56 F F F 13 260-279 33 33 33 60 60 60 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 32 32 32 57 57 57 F F F 19 380-399 28 28 28 48 48 48 F F F 20 400-419 32 32 32 55 55 55 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 30 30 30 51 51 51 F F F 28 560-579 24 24 24 39 39 39 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 33, Max flush: 60 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.839413677Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:15.890918744Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.891363394Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.928131039Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.928836031Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.928878849Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.928896575Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.928914171Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"Crucible 0fa13a71-eac6-4f34-af27-ece496c98b74 has session id: 26edb285-5d59-4525-be6e-c93e5042243d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.929934989Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Upstairs opts: Upstairs UUID: 0fa13a71-eac6-4f34-af27-ece496c98b74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.929983398Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Crucible stats registered with UUID: 0fa13a71-eac6-4f34-af27-ece496c98b74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930003359Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:15.930019913Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930745267Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930774948Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"0fa13a71-eac6-4f34-af27-ece496c98b74 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930912489Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930934404Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930970288Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.930987312Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931005209Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931021893Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931038858Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.9310545Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931120956Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931159815Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63559)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931290503Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38585)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931468036Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33480)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.931811589Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.932360111Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.932570251Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.932976502Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.933075495Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.933096698Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.933521587Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934023554Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934047213Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[0]R flush_numbers[0..12]: [29, 53, 60, 58, 56, 49, 37, 58, 58, 59, 59, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.93434389Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[0]R generation[0..12]: [20, 30, 33, 32, 32, 28, 24, 32, 32, 32, 32, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.93436822Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934385375Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[1]R flush_numbers[0..12]: [29, 53, 60, 58, 56, 49, 37, 58, 58, 59, 59, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934402109Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[1]R generation[0..12]: [20, 30, 33, 32, 32, 28, 24, 32, 32, 32, 32, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934418733Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934435247Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[2]R flush_numbers[0..12]: [29, 53, 60, 58, 56, 49, 37, 58, 58, 59, 59, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.93445161Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[2]R generation[0..12]: [20, 30, 33, 32, 32, 28, 24, 32, 32, 32, 32, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934467613Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934484217Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934504759Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934520451Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934535532Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934551044Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"":"downstairs","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934566956Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934581476Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"0","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934597338Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"1","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:04:15.93461304Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"client":"2","":"downstairs","session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934642471Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934657241Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"0fa13a71-eac6-4f34-af27-ece496c98b74 is now active with session: 26edb285-5d59-4525-be6e-c93e5042243d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934672552Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934686851Z","hostname":"bmat-EVT22200007-0000589c","pid":1323,"session_id":"26edb285-5d59-4525-be6e-c93e5042243d"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934701091Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} {"msg":"Activated sub_volume 0fa13a71-eac6-4f34-af27-ece496c98b74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:15.934714779Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 16384 data: 4 5 5 5 02/30 Write block 561 len 8192 data: 4 4 03/30 Read block 183 len 4096 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 396 len 28672 data: 5 3 4 4 6 4 4 08/30 Write block 241 len 40960 data: 2 3 3 3 3 3 3 3 3 4 09/30 Write block 288 len 40960 data: 3 3 2 2 2 2 2 4 5 5 10/30 Write block 335 len 24576 data: 4 4 3 3 4 4 11/30 Write block 547 len 16384 data: 2 2 2 2 12/30 Read block 475 len 20480 13/30 Write block 131 len 40960 data: 3 2 2 2 2 3 4 4 5 6 14/30 Read block 559 len 4096 15/30 Read block 345 len 36864 16/30 Read block 487 len 8192 17/30 Write block 542 len 28672 data: 4 4 6 3 3 3 3 18/30 Write block 232 len 12288 data: 3 3 3 19/30 Write block 240 len 12288 data: 2 3 4 20/30 Read block 371 len 4096 21/30 Write block 255 len 40960 data: 5 4 4 4 3 2 3 4 4 4 22/30 Read block 360 len 12288 23/30 Write block 261 len 4096 data: 4 24/30 Read block 411 len 32768 25/30 Write block 186 len 8192 data: 2 2 26/30 Read block 120 len 24576 27/30 Read block 356 len 28672 28/30 Write block 356 len 20480 data: 3 4 4 3 3 29/30 Read block 539 len 40960 30/30 Read block 430 len 16384 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 24 23 23 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:15.9609137Z","hostname":"bmat-EVT22200007-0000589c","pid":1323} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 32 32 32 58 58 58 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 24 24 24 37 37 37 T T T 7 140-159 32 32 32 58 58 58 T T T 8 160-179 32 32 32 58 58 58 F F F 9 180-199 32 32 32 59 59 59 T T T 10 200-219 32 32 32 59 59 59 F F F 11 220-239 18 18 18 25 25 25 T T T 12 240-259 32 32 32 56 56 56 T T T 13 260-279 33 33 33 60 60 60 T T T 14 280-299 22 22 22 34 34 34 T T T 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 T T T 17 340-359 24 24 24 39 39 39 T T T 18 360-379 32 32 32 57 57 57 T T T 19 380-399 28 28 28 48 48 48 T T T 20 400-419 32 32 32 55 55 55 T T T 21 420-439 24 24 24 39 39 39 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 30 30 30 51 51 51 T T T 28 560-579 34 34 34 61 61 61 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 34, Max flush: 61 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 32 32 32 58 58 58 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 28 28 28 49 49 49 F F F 6 120-139 35 35 35 62 62 62 F F F 7 140-159 35 35 35 62 62 62 F F F 8 160-179 32 32 32 58 58 58 F F F 9 180-199 35 35 35 62 62 62 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 35 35 35 62 62 62 F F F 13 260-279 35 35 35 62 62 62 F F F 14 280-299 35 35 35 62 62 62 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 35 35 35 62 62 62 F F F 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 F F F 19 380-399 35 35 35 62 62 62 F F F 20 400-419 35 35 35 62 62 62 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 35 35 35 62 62 62 F F F 28 560-579 34 34 34 61 61 61 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 35, Max flush: 62 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.367494261Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:20.418216907Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.418656035Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.455006967Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.455704394Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.455745569Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.455772253Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.455791362Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"Crucible b10d44f5-9113-4b00-b3fa-73e3dedd089d has session id: 10cacc6c-c183-4abb-a86e-bdcb87216211","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.456945774Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Upstairs opts: Upstairs UUID: b10d44f5-9113-4b00-b3fa-73e3dedd089d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.456994283Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Crucible stats registered with UUID: b10d44f5-9113-4b00-b3fa-73e3dedd089d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.457013152Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:20.457029235Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.457831276Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.457862049Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"b10d44f5-9113-4b00-b3fa-73e3dedd089d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.457964589Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.457984449Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458309574Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458368605Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458387794Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458404428Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458421864Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458444571Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458679501Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458698861Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"ds_connection connected from Ok(127.0.0.1:41853)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458716868Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"ds_connection connected from Ok(127.0.0.1:41979)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.458735656Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"ds_connection connected from Ok(127.0.0.1:65069)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.45875236Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.459412832Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.459446941Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.459464507Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.460119668Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.460205974Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.460543405Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.460881466Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.460970939Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[0]R flush_numbers[0..12]: [29, 53, 61, 58, 56, 49, 62, 62, 58, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461304492Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[0]R generation[0..12]: [20, 30, 34, 32, 32, 28, 35, 35, 32, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461334433Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.46135252Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[1]R flush_numbers[0..12]: [29, 53, 61, 58, 56, 49, 62, 62, 58, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461369164Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[1]R generation[0..12]: [20, 30, 34, 32, 32, 28, 35, 35, 32, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461385868Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461402623Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[2]R flush_numbers[0..12]: [29, 53, 61, 58, 56, 49, 62, 62, 58, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461418715Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[2]R generation[0..12]: [20, 30, 34, 32, 32, 28, 35, 35, 32, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461434909Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461451563Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461467175Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.46148461Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461500082Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461515794Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"":"downstairs","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461531316Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461545505Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"0","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461565025Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"1","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461580917Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"client":"2","":"downstairs","session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461597131Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.46161175Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"b10d44f5-9113-4b00-b3fa-73e3dedd089d is now active with session: 10cacc6c-c183-4abb-a86e-bdcb87216211","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461627072Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461641862Z","hostname":"bmat-EVT22200007-0000589c","pid":1331,"session_id":"10cacc6c-c183-4abb-a86e-bdcb87216211"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.461656272Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} {"msg":"Activated sub_volume b10d44f5-9113-4b00-b3fa-73e3dedd089d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:20.46166998Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 382 len 40960 data: 2 3 3 4 4 7 7 7 7 8 03/30 Read block 224 len 28672 04/30 Write block 43 len 16384 data: 5 4 3 5 05/30 Read block 37 len 20480 06/30 Flush 07/30 Write block 113 len 32768 data: 4 3 4 4 4 3 3 3 08/30 Read block 434 len 8192 09/30 Read block 482 len 24576 10/30 Read block 554 len 16384 11/30 Read block 7 len 12288 12/30 Read block 544 len 24576 13/30 Read block 51 len 32768 14/30 Read block 167 len 4096 15/30 Write block 261 len 40960 data: 5 5 5 5 5 7 6 6 5 5 16/30 Read block 513 len 32768 17/30 Write block 158 len 12288 data: 3 5 6 18/30 Read block 218 len 24576 19/30 Read block 482 len 12288 20/30 Write block 113 len 16384 data: 5 4 5 5 21/30 Read block 323 len 12288 22/30 Write block 557 len 20480 data: 6 5 5 3 5 23/30 Write block 72 len 16384 data: 2 2 2 3 24/30 Read block 438 len 16384 25/30 Read block 585 len 36864 26/30 Read block 115 len 12288 27/30 Read block 220 len 40960 28/30 Write block 455 len 32768 data: 2 4 4 4 4 5 6 7 29/30 Read block 441 len 24576 30/30 Write block 403 len 40960 data: 5 6 5 4 4 6 6 5 5 6 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:20.492410463Z","hostname":"bmat-EVT22200007-0000589c","pid":1331} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 36 36 36 64 64 64 F F F 3 060-079 32 32 32 58 58 58 T T T 4 080-099 32 32 32 56 56 56 F F F 5 100-119 28 28 28 49 49 49 T T T 6 120-139 35 35 35 62 62 62 T T T 7 140-159 35 35 35 62 62 62 T T T 8 160-179 32 32 32 58 58 58 T T T 9 180-199 35 35 35 62 62 62 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 35 35 35 62 62 62 F F F 13 260-279 35 35 35 62 62 62 T T T 14 280-299 35 35 35 62 62 62 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 35 35 35 62 62 62 F F F 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 35 35 35 62 62 62 T T T 21 420-439 24 24 24 39 39 39 F F F 22 440-459 32 32 32 58 58 58 T T T 23 460-479 32 32 32 56 56 56 T T T 24 480-499 30 30 30 52 52 52 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 35 35 35 62 62 62 T T T 28 560-579 34 34 34 61 61 61 T T T 29 580-599 31 31 31 54 54 54 F F F Max gen: 36, Max flush: 64 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 36 36 36 64 64 64 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 37 37 37 65 65 65 F F F 6 120-139 37 37 37 65 65 65 F F F 7 140-159 37 37 37 65 65 65 F F F 8 160-179 37 37 37 65 65 65 F F F 9 180-199 35 35 35 62 62 62 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 35 35 35 62 62 62 F F F 13 260-279 37 37 37 65 65 65 F F F 14 280-299 35 35 35 62 62 62 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 35 35 35 62 62 62 F F F 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 37 37 37 65 65 65 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 F F F 24 480-499 30 30 30 52 52 52 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 37, Max flush: 65 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.877626743Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.931866034Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.933491429Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93358958Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.933611886Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.933631416Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"Crucible dce7e819-95fc-4cf5-bb15-7298f86a076d has session id: 3a9f9d9f-138d-42c0-9651-af25fd97c988","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.934748241Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Upstairs opts: Upstairs UUID: dce7e819-95fc-4cf5-bb15-7298f86a076d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.934838616Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Crucible stats registered with UUID: dce7e819-95fc-4cf5-bb15-7298f86a076d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.934858937Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:24.93487471Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935553899Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935592869Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"dce7e819-95fc-4cf5-bb15-7298f86a076d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935726282Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935748317Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93577936Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935796014Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935813731Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935830174Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935846969Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93586244Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.935879375Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.936651625Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"ds_connection connected from Ok(127.0.0.1:55917)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.936687759Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"ds_connection connected from Ok(127.0.0.1:48238)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.936705766Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"ds_connection connected from Ok(127.0.0.1:43250)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.936722841Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93707932Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.937265641Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.937918317Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.938180443Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.938203731Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.938393809Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.938743325Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.938855965Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[0]R flush_numbers[0..12]: [29, 53, 64, 65, 56, 65, 65, 65, 65, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939042857Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[0]R generation[0..12]: [20, 30, 36, 37, 32, 37, 37, 37, 37, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93906861Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939086356Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[1]R flush_numbers[0..12]: [29, 53, 64, 65, 56, 65, 65, 65, 65, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93910272Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[1]R generation[0..12]: [20, 30, 36, 37, 32, 37, 37, 37, 37, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939118592Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939135367Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[2]R flush_numbers[0..12]: [29, 53, 64, 65, 56, 65, 65, 65, 65, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939154275Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[2]R generation[0..12]: [20, 30, 36, 37, 32, 37, 37, 37, 37, 35, 32, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939175669Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939192724Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939208035Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939223407Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939238437Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939253729Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"":"downstairs","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.9392689Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.93928347Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"0","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939299332Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"1","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939314954Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"client":"2","":"downstairs","session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939330476Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939344865Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"dce7e819-95fc-4cf5-bb15-7298f86a076d is now active with session: 3a9f9d9f-138d-42c0-9651-af25fd97c988","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939360157Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939374025Z","hostname":"bmat-EVT22200007-0000589c","pid":1339,"session_id":"3a9f9d9f-138d-42c0-9651-af25fd97c988"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939388755Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} {"msg":"Activated sub_volume dce7e819-95fc-4cf5-bb15-7298f86a076d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:24.939402513Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 57 len 4096 data: 5 02/30 Write block 225 len 36864 data: 6 4 2 3 2 2 3 4 4 03/30 Read block 214 len 20480 04/30 Write block 305 len 8192 data: 3 4 05/30 Write block 280 len 16384 data: 5 5 4 3 06/30 Read block 159 len 40960 07/30 Write block 143 len 40960 data: 7 8 8 7 3 3 2 2 2 2 08/30 Write block 523 len 12288 data: 3 3 3 09/30 Read block 297 len 16384 10/30 Write block 296 len 28672 data: 6 6 6 5 5 5 5 11/30 Read block 65 len 8192 12/30 Read block 435 len 24576 13/30 Read block 560 len 20480 14/30 Read block 380 len 32768 15/30 Read block 328 len 36864 16/30 Read block 144 len 28672 17/30 Read block 402 len 28672 18/30 Read block 352 len 28672 19/30 Write block 256 len 8192 data: 5 5 20/30 Read block 25 len 32768 21/30 Write block 297 len 16384 data: 7 7 6 6 22/30 Read block 517 len 36864 23/30 Read block 488 len 24576 24/30 Write block 166 len 40960 data: 4 6 6 5 5 5 6 5 4 3 25/30 Write block 494 len 24576 data: 2 3 3 3 3 4 26/30 Read block 475 len 4096 27/30 Read block 159 len 16384 28/30 Write block 320 len 32768 data: 4 3 4 2 2 2 2 4 29/30 Read block 279 len 36864 30/30 Write block 324 len 16384 data: 3 3 3 5 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:24.967629384Z","hostname":"bmat-EVT22200007-0000589c","pid":1339} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 36 36 36 64 64 64 T T T 3 060-079 37 37 37 65 65 65 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 37 37 37 65 65 65 F F F 6 120-139 37 37 37 65 65 65 F F F 7 140-159 37 37 37 65 65 65 T T T 8 160-179 37 37 37 65 65 65 T T T 9 180-199 35 35 35 62 62 62 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 35 35 35 62 62 62 T T T 12 240-259 35 35 35 62 62 62 T T T 13 260-279 37 37 37 65 65 65 F F F 14 280-299 35 35 35 62 62 62 T T T 15 300-319 30 30 30 52 52 52 T T T 16 320-339 35 35 35 62 62 62 T T T 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 37 37 37 65 65 65 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 F F F 24 480-499 30 30 30 52 52 52 T T T 25 500-519 33 33 33 60 60 60 F F F 26 520-539 30 30 30 51 51 51 T T T 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 37, Max flush: 65 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 32 32 32 56 56 56 F F F 5 100-119 37 37 37 65 65 65 F F F 6 120-139 37 37 37 65 65 65 F F F 7 140-159 38 38 38 66 66 66 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 35 35 35 62 62 62 F F F 10 200-219 32 32 32 59 59 59 F F F 11 220-239 38 38 38 66 66 66 F F F 12 240-259 38 38 38 66 66 66 F F F 13 260-279 37 37 37 65 65 65 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 38 38 38 66 66 66 F F F 16 320-339 38 38 38 66 66 66 F F F 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 37 37 37 65 65 65 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 F F F 24 480-499 38 38 38 66 66 66 F F F 25 500-519 33 33 33 60 60 60 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 38, Max flush: 66 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.325446277Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.379919156Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.380682769Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.380746489Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.380767522Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.380785659Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"Crucible ba5bed19-0067-4a7f-87a5-3fc2531e394b has session id: 2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.381802479Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Upstairs opts: Upstairs UUID: ba5bed19-0067-4a7f-87a5-3fc2531e394b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.381854436Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Crucible stats registered with UUID: ba5bed19-0067-4a7f-87a5-3fc2531e394b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.381875789Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:29.381892193Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.382619701Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.382648611Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"ba5bed19-0067-4a7f-87a5-3fc2531e394b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.382933063Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.382954005Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.382992925Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383012385Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383030662Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383052988Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383070574Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383184888Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383207304Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383225391Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:35683)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383591189Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:54482)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.38362579Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:35410)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.383661453Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.384757636Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.384793058Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.38481345Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.385528382Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.38569841Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.385740045Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386356637Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386419155Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[0]R flush_numbers[0..12]: [29, 53, 66, 65, 56, 65, 65, 66, 66, 62, 59, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386572128Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[0]R generation[0..12]: [20, 30, 38, 37, 32, 37, 37, 38, 38, 35, 32, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386597961Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386616068Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[1]R flush_numbers[0..12]: [29, 53, 66, 65, 56, 65, 65, 66, 66, 62, 59, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386633333Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[1]R generation[0..12]: [20, 30, 38, 37, 32, 37, 37, 38, 38, 35, 32, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386650067Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386666952Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[2]R flush_numbers[0..12]: [29, 53, 66, 65, 56, 65, 65, 66, 66, 62, 59, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386684568Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[2]R generation[0..12]: [20, 30, 38, 37, 32, 37, 37, 38, 38, 35, 32, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386700551Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386717425Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Max found gen is 39","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386733147Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Generation requested: 40 >= found:39","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.3867489Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386764511Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386779953Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"":"downstairs","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386805475Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386820336Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"0","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386836559Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"1","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386852301Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"client":"2","":"downstairs","session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386868344Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386882914Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"ba5bed19-0067-4a7f-87a5-3fc2531e394b is now active with session: 2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386898185Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386912294Z","hostname":"bmat-EVT22200007-0000589c","pid":1347,"session_id":"2e6dd8e1-89d1-4cfe-9e48-78c7e9faf417"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386927094Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} {"msg":"Activated sub_volume ba5bed19-0067-4a7f-87a5-3fc2531e394b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:29.386948117Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 504 len 28672 data: 3 4 4 4 4 3 3 02/30 Write block 315 len 24576 data: 4 5 5 5 4 5 03/30 Read block 379 len 20480 04/30 Read block 550 len 32768 05/30 Read block 202 len 16384 06/30 Write block 290 len 32768 data: 3 3 3 3 3 5 7 8 07/30 Write block 138 len 32768 data: 5 6 7 6 6 8 9 9 08/30 Read block 517 len 32768 09/30 Write block 427 len 28672 data: 3 3 3 3 4 6 7 10/30 Write block 101 len 8192 data: 4 5 11/30 Read block 93 len 4096 12/30 Flush 13/30 Write block 317 len 20480 data: 6 6 5 6 4 14/30 Write block 583 len 12288 data: 4 4 5 15/30 Write block 87 len 16384 data: 4 3 4 5 16/30 Read block 594 len 12288 17/30 Read block 309 len 24576 18/30 Read block 202 len 28672 19/30 Write block 198 len 28672 data: 4 3 3 3 3 4 3 20/30 Write block 274 len 28672 data: 4 4 4 4 5 5 6 21/30 Read block 574 len 32768 22/30 Write block 371 len 36864 data: 6 5 8 8 7 7 4 3 2 23/30 Read block 74 len 24576 24/30 Read block 428 len 4096 25/30 Read block 406 len 16384 26/30 Read block 540 len 24576 27/30 Write block 185 len 40960 data: 2 3 3 2 2 4 4 5 5 5 28/30 Write block 330 len 20480 data: 4 5 5 4 5 29/30 Write block 410 len 16384 data: 6 6 7 6 30/30 Write block 32 len 16384 data: 4 3 5 4 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:29.414625474Z","hostname":"bmat-EVT22200007-0000589c","pid":1347} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 30 30 30 53 53 53 T T T 2 040-059 38 38 38 66 66 66 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 32 32 32 56 56 56 T T T 5 100-119 40 40 40 67 67 67 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 40 40 40 67 67 67 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 35 35 35 62 62 62 T T T 10 200-219 32 32 32 59 59 59 T T T 11 220-239 38 38 38 66 66 66 F F F 12 240-259 38 38 38 66 66 66 F F F 13 260-279 37 37 37 65 65 65 T T T 14 280-299 40 40 40 67 67 67 T T T 15 300-319 40 40 40 67 67 67 T T T 16 320-339 40 40 40 67 67 67 T T T 17 340-359 35 35 35 62 62 62 F F F 18 360-379 35 35 35 62 62 62 T T T 19 380-399 36 36 36 64 64 64 F F F 20 400-419 37 37 37 65 65 65 T T T 21 420-439 40 40 40 67 67 67 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 F F F 24 480-499 38 38 38 66 66 66 F F F 25 500-519 40 40 40 67 67 67 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 54 54 54 T T T Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 41 41 41 68 68 68 F F F 5 100-119 40 40 40 67 67 67 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 40 40 40 67 67 67 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 41 41 41 68 68 68 F F F 11 220-239 38 38 38 66 66 66 F F F 12 240-259 38 38 38 66 66 66 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 35 35 35 62 62 62 F F F 18 360-379 41 41 41 68 68 68 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 40 40 40 67 67 67 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 F F F 24 480-499 38 38 38 66 66 66 F F F 25 500-519 40 40 40 67 67 67 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.77781138Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.831985818Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.832723758Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.832770634Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.832789032Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.832806026Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"Crucible 47a65661-6469-4bb4-b450-946b921a10f6 has session id: 0f4452b1-df19-4ef5-bb58-4d8b28c26a53","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.833687189Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Upstairs opts: Upstairs UUID: 47a65661-6469-4bb4-b450-946b921a10f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.833735107Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Crucible stats registered with UUID: 47a65661-6469-4bb4-b450-946b921a10f6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.833753314Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:33.833775089Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834524121Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834552399Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"47a65661-6469-4bb4-b450-946b921a10f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834677826Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834697847Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834727668Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.834744242Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835028904Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835062403Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835215225Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835241239Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835258504Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835275088Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:36060)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835457922Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:60464)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835514027Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"ds_connection connected from Ok(127.0.0.1:56757)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.835533677Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.836734213Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.836818476Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.836838426Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.837132999Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.837371838Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.837693205Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.83782724Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838331692Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[0]R flush_numbers[0..12]: [29, 68, 66, 65, 68, 67, 67, 67, 66, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838483512Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[0]R generation[0..12]: [20, 41, 38, 37, 41, 40, 40, 40, 38, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838508563Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838528294Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[1]R flush_numbers[0..12]: [29, 68, 66, 65, 68, 67, 67, 67, 66, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838546832Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[1]R generation[0..12]: [20, 41, 38, 37, 41, 40, 40, 40, 38, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838565089Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838583998Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[2]R flush_numbers[0..12]: [29, 68, 66, 65, 68, 67, 67, 67, 66, 68, 68, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838608528Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[2]R generation[0..12]: [20, 41, 38, 37, 41, 40, 40, 40, 38, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838625573Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838642457Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838657829Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838673481Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838688482Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838703562Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"":"downstairs","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838718623Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838732832Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"0","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838748775Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"1","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838764086Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"client":"2","":"downstairs","session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838779899Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838794448Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"47a65661-6469-4bb4-b450-946b921a10f6 is now active with session: 0f4452b1-df19-4ef5-bb58-4d8b28c26a53","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.83880975Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838823728Z","hostname":"bmat-EVT22200007-0000589c","pid":1355,"session_id":"0f4452b1-df19-4ef5-bb58-4d8b28c26a53"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838838248Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} {"msg":"Activated sub_volume 47a65661-6469-4bb4-b450-946b921a10f6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:33.838857167Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 8192 data: 6 7 02/30 Read block 482 len 16384 03/30 Write block 106 len 40960 data: 5 4 5 4 3 2 4 6 5 6 04/30 Write block 82 len 24576 data: 5 3 4 4 4 5 05/30 Read block 402 len 20480 06/30 Write block 373 len 8192 data: 9 9 07/30 Write block 216 len 28672 data: 4 6 4 5 5 5 7 08/30 Read block 331 len 40960 09/30 Write block 172 len 12288 data: 7 6 5 10/30 Read block 398 len 32768 11/30 Read block 430 len 8192 12/30 Read block 328 len 4096 13/30 Read block 455 len 40960 14/30 Read block 300 len 20480 15/30 Read block 151 len 8192 16/30 Read block 56 len 8192 17/30 Read block 81 len 4096 18/30 Write block 350 len 32768 data: 2 2 3 3 3 3 4 5 19/30 Read block 113 len 32768 20/30 Read block 150 len 24576 21/30 Read block 455 len 12288 22/30 Flush 23/30 Write block 88 len 32768 data: 4 5 6 5 5 6 5 5 24/30 Write block 111 len 32768 data: 3 5 7 6 7 6 5 4 25/30 Write block 464 len 12288 data: 9 9 7 26/30 Write block 439 len 4096 data: 4 27/30 Write block 243 len 16384 data: 4 4 4 4 28/30 Read block 422 len 16384 29/30 Read block 565 len 40960 30/30 Read block 194 len 40960 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:33.8667039Z","hostname":"bmat-EVT22200007-0000589c","pid":1355} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 42 42 42 69 69 69 T T T 5 100-119 42 42 42 69 69 69 T T T 6 120-139 40 40 40 67 67 67 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 38 38 38 66 66 66 T T T 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 42 42 42 69 69 69 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 40 40 40 67 67 67 T T T 22 440-459 37 37 37 65 65 65 F F F 23 460-479 37 37 37 65 65 65 T T T 24 480-499 38 38 38 66 66 66 F F F 25 500-519 40 40 40 67 67 67 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 42, Max flush: 69 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 29 29 29 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 37 37 37 65 65 65 F F F 4 080-099 43 43 43 70 70 70 F F F 5 100-119 43 43 43 70 70 70 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 43 43 43 70 70 70 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 42 42 42 69 69 69 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 43 43 43 70 70 70 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 38 38 38 66 66 66 F F F 25 500-519 40 40 40 67 67 67 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 37 37 37 65 65 65 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 43, Max flush: 70 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.189126347Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.241848914Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.242630604Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.242694565Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.242716469Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.242734055Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"Crucible a479f390-368f-41dd-8bac-c43ffce194fc has session id: c5d58881-e169-4d1b-90c8-9a218f4bf926","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.243513851Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Upstairs opts: Upstairs UUID: a479f390-368f-41dd-8bac-c43ffce194fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.24358191Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Crucible stats registered with UUID: a479f390-368f-41dd-8bac-c43ffce194fc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.243602883Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:38.243618755Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244804451Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244836908Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"a479f390-368f-41dd-8bac-c43ffce194fc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244869504Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244886098Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.24491614Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244934497Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244951242Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244966834Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.244984019Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245079875Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245118324Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245276978Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"ds_connection connected from Ok(127.0.0.1:37528)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245682698Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"ds_connection connected from Ok(127.0.0.1:35753)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245795449Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"ds_connection connected from Ok(127.0.0.1:34972)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.245851654Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.246301775Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.246537066Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.246743949Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.247494916Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.247529096Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.247623178Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.24838179Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248551508Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[0]R flush_numbers[0..12]: [29, 68, 66, 65, 70, 70, 67, 69, 69, 68, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248655651Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[0]R generation[0..12]: [20, 41, 38, 37, 43, 43, 40, 42, 42, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248678237Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248705252Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[1]R flush_numbers[0..12]: [29, 68, 66, 65, 70, 70, 67, 69, 69, 68, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248723209Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[1]R generation[0..12]: [20, 41, 38, 37, 43, 43, 40, 42, 42, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248740204Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248757669Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[2]R flush_numbers[0..12]: [29, 68, 66, 65, 70, 70, 67, 69, 69, 68, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248774674Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[2]R generation[0..12]: [20, 41, 38, 37, 43, 43, 40, 42, 42, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248791278Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248808503Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248824466Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248840759Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248856421Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248872124Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"":"downstairs","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248888126Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248903017Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"0","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.24891945Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"1","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248935614Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"client":"2","":"downstairs","session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248955644Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.248971196Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"a479f390-368f-41dd-8bac-c43ffce194fc is now active with session: c5d58881-e169-4d1b-90c8-9a218f4bf926","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:04:38.248986898Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.249016008Z","hostname":"bmat-EVT22200007-0000589c","pid":1363,"session_id":"c5d58881-e169-4d1b-90c8-9a218f4bf926"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.249032171Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} {"msg":"Activated sub_volume a479f390-368f-41dd-8bac-c43ffce194fc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:38.249046059Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 12288 data: 4 4 3 02/30 Read block 484 len 28672 03/30 Read block 311 len 40960 04/30 Read block 72 len 36864 05/30 Read block 594 len 8192 06/30 Flush 07/30 Flush 08/30 Write block 14 len 28672 data: 2 2 2 2 2 2 2 09/30 Write block 96 len 8192 data: 5 5 10/30 Read block 438 len 32768 11/30 Write block 67 len 20480 data: 2 2 2 2 2 12/30 Write block 235 len 24576 data: 3 3 3 3 2 3 13/30 Flush 14/30 Write block 184 len 36864 data: 2 3 4 4 3 3 5 5 6 15/30 Read block 260 len 40960 16/30 Write block 514 len 36864 data: 8 7 7 7 6 3 3 3 3 17/30 Write block 490 len 20480 data: 2 2 2 2 3 18/30 Read block 77 len 24576 19/30 Write block 60 len 24576 data: 5 4 3 3 2 2 20/30 Read block 85 len 16384 21/30 Read block 342 len 16384 22/30 Write block 336 len 36864 data: 5 4 4 5 5 2 3 3 4 23/30 Write block 124 len 16384 data: 2 2 2 2 24/30 Write block 256 len 16384 data: 6 6 5 4 25/30 Write block 116 len 20480 data: 7 6 5 4 4 26/30 Read block 497 len 12288 27/30 Write block 110 len 8192 data: 4 4 28/30 Write block 487 len 12288 data: 5 5 2 29/30 Read block 162 len 40960 30/30 Read block 1 len 12288 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 17 17 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:38.27569899Z","hostname":"bmat-EVT22200007-0000589c","pid":1363} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 44 44 44 73 73 73 T T T 4 080-099 44 44 44 73 73 73 F F F 5 100-119 43 43 43 70 70 70 T T T 6 120-139 40 40 40 67 67 67 T T T 7 140-159 42 42 42 69 69 69 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 41 41 41 68 68 68 T T T 10 200-219 42 42 42 69 69 69 F F F 11 220-239 44 44 44 73 73 73 F F F 12 240-259 44 44 44 73 73 73 T T T 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 41 41 41 68 68 68 T T T 17 340-359 42 42 42 69 69 69 T T T 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 43 43 43 70 70 70 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 38 38 38 66 66 66 T T T 25 500-519 40 40 40 67 67 67 T T T 26 520-539 38 38 38 66 66 66 T T T 27 540-559 44 44 44 71 71 71 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 44, Max flush: 73 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 45 45 45 74 74 74 F F F 4 080-099 44 44 44 73 73 73 F F F 5 100-119 45 45 45 74 74 74 F F F 6 120-139 45 45 45 74 74 74 F F F 7 140-159 42 42 42 69 69 69 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 45 45 45 74 74 74 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 44 44 44 73 73 73 F F F 12 240-259 45 45 45 74 74 74 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 45 45 45 74 74 74 F F F 17 340-359 45 45 45 74 74 74 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 43 43 43 70 70 70 F F F 22 440-459 37 37 37 65 65 65 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 45 45 45 74 74 74 F F F 25 500-519 45 45 45 74 74 74 F F F 26 520-539 45 45 45 74 74 74 F F F 27 540-559 44 44 44 71 71 71 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 45, Max flush: 74 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.635580987Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.69016353Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691034833Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691115478Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691134657Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691153165Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"Crucible 78148f34-85f3-4ffa-a66e-1402706f05c3 has session id: 2da41ac5-5203-47a3-9d3a-314cffdfc1e3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691897197Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Upstairs opts: Upstairs UUID: 78148f34-85f3-4ffa-a66e-1402706f05c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.691984325Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Crucible stats registered with UUID: 78148f34-85f3-4ffa-a66e-1402706f05c3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.692008605Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:42.692041141Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.692802459Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.692834174Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"78148f34-85f3-4ffa-a66e-1402706f05c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.692956624Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.692980072Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693019303Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693212267Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693274314Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693293484Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693311881Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693327844Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693345179Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693361894Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49464)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693456417Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64754)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693495757Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45900)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.693816704Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.6944306Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.694808804Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.695111082Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.69531499Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.695496922Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.695521632Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696353955Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696444751Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[0]R flush_numbers[0..12]: [73, 73, 66, 74, 73, 74, 74, 69, 69, 74, 69, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696464702Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[0]R generation[0..12]: [44, 44, 38, 45, 44, 45, 45, 42, 42, 45, 42, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696482759Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696500565Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[1]R flush_numbers[0..12]: [73, 73, 66, 74, 73, 74, 74, 69, 69, 74, 69, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.69651763Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[1]R generation[0..12]: [44, 44, 38, 45, 44, 45, 45, 42, 42, 45, 42, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696534344Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.69655169Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[2]R flush_numbers[0..12]: [73, 73, 66, 74, 73, 74, 74, 69, 69, 74, 69, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696568875Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[2]R generation[0..12]: [44, 44, 38, 45, 44, 45, 45, 42, 42, 45, 42, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696585689Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696602915Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:04:42.696729754Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696814387Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696831633Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696847305Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"":"downstairs","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696863478Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696877977Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"0","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696894241Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"1","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696910193Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"client":"2","":"downstairs","session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696926537Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696941487Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"78148f34-85f3-4ffa-a66e-1402706f05c3 is now active with session: 2da41ac5-5203-47a3-9d3a-314cffdfc1e3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696957179Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.696971609Z","hostname":"bmat-EVT22200007-0000589c","pid":1371,"session_id":"2da41ac5-5203-47a3-9d3a-314cffdfc1e3"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.69698672Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} {"msg":"Activated sub_volume 78148f34-85f3-4ffa-a66e-1402706f05c3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:42.697000438Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 144 len 32768 data: 10 10 8 4 4 3 3 3 02/30 Read block 99 len 28672 03/30 Write block 237 len 20480 data: 4 4 3 4 4 04/30 Flush 05/30 Flush 06/30 Write block 518 len 4096 data: 7 07/30 Read block 98 len 36864 08/30 Write block 171 len 40960 data: 6 8 7 6 4 4 3 5 5 3 09/30 Read block 250 len 28672 10/30 Read block 489 len 20480 11/30 Write block 521 len 20480 data: 4 4 4 4 4 12/30 Read block 341 len 12288 13/30 Write block 438 len 28672 data: 6 5 4 4 5 4 4 14/30 Read block 288 len 40960 15/30 Write block 195 len 24576 data: 5 4 4 5 4 4 16/30 Write block 294 len 8192 data: 4 6 17/30 Write block 60 len 40960 data: 6 5 4 4 3 3 2 3 3 3 18/30 Write block 99 len 40960 data: 6 6 5 6 5 6 5 6 5 6 19/30 Read block 501 len 20480 20/30 Write block 185 len 16384 data: 4 5 5 4 21/30 Read block 437 len 28672 22/30 Read block 549 len 4096 23/30 Flush 24/30 Write block 158 len 32768 data: 4 7 8 7 6 6 6 4 25/30 Read block 564 len 12288 26/30 Read block 436 len 32768 27/30 Read block 80 len 24576 28/30 Read block 241 len 28672 29/30 Write block 536 len 36864 data: 4 5 6 5 5 5 5 5 7 30/30 Read block 579 len 40960 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 7 7 5 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:42.726143303Z","hostname":"bmat-EVT22200007-0000589c","pid":1371} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 45 45 45 74 74 74 F F F 7 140-159 46 46 46 75 75 75 T T T 8 160-179 46 46 46 77 77 77 T T T 9 180-199 46 46 46 77 77 77 F F F 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 46 46 46 75 75 75 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 46 46 46 77 77 77 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 45 45 45 74 74 74 F F F 17 340-359 45 45 45 74 74 74 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 46 46 46 77 77 77 F F F 22 440-459 46 46 46 77 77 77 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 45 45 45 74 74 74 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 46 46 46 77 77 77 T T T 27 540-559 44 44 44 71 71 71 T T T 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 46, Max flush: 77 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 45 45 45 74 74 74 F F F 7 140-159 47 47 47 78 78 78 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 46 46 46 77 77 77 F F F 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 46 46 46 75 75 75 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 46 46 46 77 77 77 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 45 45 45 74 74 74 F F F 17 340-359 45 45 45 74 74 74 F F F 18 360-379 42 42 42 69 69 69 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 46 46 46 77 77 77 F F F 22 440-459 46 46 46 77 77 77 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 45 45 45 74 74 74 F F F 25 500-519 46 46 46 77 77 77 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 47 47 47 78 78 78 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 47, Max flush: 78 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.042167908Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.09710167Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.097918772Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.097972923Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.097991912Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.09801065Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"Crucible 041ffcda-3459-46a0-b9cf-bd58d0bdb56d has session id: 77584c25-4a83-4fcf-96fa-59846353f16f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.098848584Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Upstairs opts: Upstairs UUID: 041ffcda-3459-46a0-b9cf-bd58d0bdb56d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.098901122Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Crucible stats registered with UUID: 041ffcda-3459-46a0-b9cf-bd58d0bdb56d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.098929219Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:47.098945974Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.099874594Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.099941781Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"041ffcda-3459-46a0-b9cf-bd58d0bdb56d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.099961992Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.099979919Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100039501Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100059281Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100077188Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.10009278Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100109845Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100125517Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.10014183Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100157993Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33638)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100476705Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58495)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100516006Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47637)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.100542751Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.101455017Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.101721532Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.101956082Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.102407135Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.102659441Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103011852Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103283237Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103320734Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[0]R flush_numbers[0..12]: [73, 73, 66, 77, 77, 77, 74, 78, 78, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103586919Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[0]R generation[0..12]: [44, 44, 38, 46, 46, 46, 45, 47, 47, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103612551Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.10363141Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[1]R flush_numbers[0..12]: [73, 73, 66, 77, 77, 77, 74, 78, 78, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103649136Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[1]R generation[0..12]: [44, 44, 38, 46, 46, 46, 45, 47, 47, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103666001Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103690671Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[2]R flush_numbers[0..12]: [73, 73, 66, 77, 77, 77, 74, 78, 78, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103708347Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[2]R generation[0..12]: [44, 44, 38, 46, 46, 46, 45, 47, 47, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103725342Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103742768Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.10375869Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103774743Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103790475Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103806027Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"":"downstairs","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103822401Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103837151Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"0","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103853614Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"1","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103869778Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"client":"2","":"downstairs","session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103885991Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103900821Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"041ffcda-3459-46a0-b9cf-bd58d0bdb56d is now active with session: 77584c25-4a83-4fcf-96fa-59846353f16f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103916714Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103931364Z","hostname":"bmat-EVT22200007-0000589c","pid":1379,"session_id":"77584c25-4a83-4fcf-96fa-59846353f16f"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103949851Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} {"msg":"Activated sub_volume 041ffcda-3459-46a0-b9cf-bd58d0bdb56d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:47.103964101Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 8192 data: 10 8 02/30 Flush 03/30 Write block 435 len 28672 data: 7 7 6 7 6 5 5 04/30 Write block 130 len 12288 data: 2 4 3 05/30 Read block 16 len 20480 06/30 Read block 106 len 12288 07/30 Write block 322 len 16384 data: 5 3 4 4 08/30 Flush 09/30 Write block 147 len 12288 data: 5 5 4 10/30 Read block 549 len 12288 11/30 Read block 272 len 4096 12/30 Write block 359 len 40960 data: 4 4 3 3 4 5 6 7 4 4 13/30 Flush 14/30 Write block 273 len 40960 data: 4 5 5 5 5 6 6 7 6 5 15/30 Write block 23 len 40960 data: 3 4 4 6 6 5 4 4 4 5 16/30 Write block 40 len 28672 data: 5 4 5 6 5 4 6 17/30 Write block 545 len 36864 data: 4 4 5 5 4 3 2 2 2 18/30 Flush 19/30 Flush 20/30 Write block 337 len 32768 data: 5 5 6 6 3 4 4 5 21/30 Write block 140 len 24576 data: 8 7 7 9 11 11 22/30 Read block 180 len 32768 23/30 Write block 159 len 36864 data: 8 9 8 7 7 7 5 5 7 24/30 Flush 25/30 Flush 26/30 Write block 259 len 4096 data: 5 27/30 Write block 500 len 20480 data: 5 5 4 3 4 28/30 Write block 420 len 24576 data: 6 4 4 4 4 4 29/30 Read block 524 len 40960 30/30 Write block 185 len 32768 data: 5 6 6 5 4 6 6 7 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:47.133001991Z","hostname":"bmat-EVT22200007-0000589c","pid":1379} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 82 82 82 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 48 48 48 80 80 80 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 48 48 48 84 84 84 F F F 9 180-199 46 46 46 77 77 77 T T T 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 46 46 46 75 75 75 T T T 13 260-279 48 48 48 82 82 82 F F F 14 280-299 48 48 48 82 82 82 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 48 48 48 80 80 80 T T T 22 440-459 48 48 48 80 80 80 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 45 45 45 74 74 74 F F F 25 500-519 46 46 46 77 77 77 T T T 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 48, Max flush: 84 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 82 82 82 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 48 48 48 80 80 80 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 48 48 48 84 84 84 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 49 49 49 85 85 85 F F F 13 260-279 48 48 48 82 82 82 F F F 14 280-299 48 48 48 82 82 82 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 48 48 48 81 81 81 F F F 19 380-399 36 36 36 64 64 64 F F F 20 400-419 41 41 41 68 68 68 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 48 48 48 80 80 80 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 45 45 45 74 74 74 F F F 25 500-519 49 49 49 85 85 85 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 49, Max flush: 85 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.700559682Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:51.753263692Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.753719695Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.791421701Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.792121563Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.792168178Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.792194813Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79221307Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"Crucible 7b434d0a-3020-42cc-9497-cc493ad382e7 has session id: ae4cbc0e-dced-4503-b9b4-35267ae9b169","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.793249621Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Upstairs opts: Upstairs UUID: 7b434d0a-3020-42cc-9497-cc493ad382e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.793301868Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Crucible stats registered with UUID: 7b434d0a-3020-42cc-9497-cc493ad382e7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.793321277Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:51.793336589Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794316704Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794346084Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"7b434d0a-3020-42cc-9497-cc493ad382e7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794527175Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79454941Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794582037Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79459851Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794616357Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79463228Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794656439Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794751814Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794772787Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794789471Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"ds_connection connected from Ok(127.0.0.1:62683)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.794978488Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"ds_connection connected from Ok(127.0.0.1:43464)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.795004421Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"ds_connection connected from Ok(127.0.0.1:61343)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.795021776Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.796134082Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.796195237Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.796779152Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.796925992Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.797014363Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.797587386Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.797728625Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.797762133Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[0]R flush_numbers[0..12]: [73, 82, 82, 77, 77, 77, 80, 84, 84, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798607843Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[0]R generation[0..12]: [44, 48, 48, 46, 46, 46, 48, 48, 48, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798632203Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798649599Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[1]R flush_numbers[0..12]: [73, 82, 82, 77, 77, 77, 80, 84, 84, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798666032Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[1]R generation[0..12]: [44, 48, 48, 46, 46, 46, 48, 48, 48, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798682336Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798698609Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[2]R flush_numbers[0..12]: [73, 82, 82, 77, 77, 77, 80, 84, 84, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798714782Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[2]R generation[0..12]: [44, 48, 48, 46, 46, 46, 48, 48, 48, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798730815Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798747379Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79876271Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798778352Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798793583Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798808664Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"":"downstairs","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798824146Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798838255Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"0","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798857875Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"1","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798873657Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"client":"2","":"downstairs","session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798889289Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798903609Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"7b434d0a-3020-42cc-9497-cc493ad382e7 is now active with session: ae4cbc0e-dced-4503-b9b4-35267ae9b169","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.79891883Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798933179Z","hostname":"bmat-EVT22200007-0000589c","pid":1387,"session_id":"ae4cbc0e-dced-4503-b9b4-35267ae9b169"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798947568Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} {"msg":"Activated sub_volume 7b434d0a-3020-42cc-9497-cc493ad382e7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:51.798961487Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 40960 data: 5 5 5 5 5 4 4 4 5 5 02/30 Read block 224 len 32768 03/30 Write block 304 len 20480 data: 3 4 5 3 3 04/30 Read block 53 len 16384 05/30 Read block 561 len 8192 06/30 Read block 527 len 28672 07/30 Read block 457 len 24576 08/30 Read block 269 len 32768 09/30 Read block 344 len 24576 10/30 Read block 388 len 8192 11/30 Read block 425 len 40960 12/30 Read block 0 len 16384 13/30 Read block 238 len 32768 14/30 Write block 405 len 40960 data: 6 5 5 7 7 7 7 8 7 5 15/30 Read block 348 len 32768 16/30 Read block 355 len 36864 17/30 Flush 18/30 Write block 569 len 16384 data: 4 6 5 5 19/30 Write block 385 len 12288 data: 5 5 8 20/30 Read block 156 len 4096 21/30 Write block 365 len 28672 data: 7 8 5 5 4 4 7 22/30 Write block 479 len 20480 data: 5 5 6 5 5 23/30 Write block 168 len 8192 data: 7 6 24/30 Write block 3 len 36864 data: 3 3 3 3 3 2 3 3 2 25/30 Read block 378 len 24576 26/30 Flush 27/30 Write block 395 len 40960 data: 6 6 4 5 5 7 5 5 6 7 28/30 Write block 126 len 32768 data: 3 3 2 2 3 5 4 3 29/30 Write block 379 len 4096 data: 3 30/30 Read block 321 len 16384 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:51.827325969Z","hostname":"bmat-EVT22200007-0000589c","pid":1387} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 87 87 87 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 82 82 82 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 48 48 48 80 80 80 T T T 7 140-159 48 48 48 84 84 84 F F F 8 160-179 50 50 50 87 87 87 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 48 48 48 82 82 82 F F F 14 280-299 48 48 48 82 82 82 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 50 50 50 87 87 87 T T T 19 380-399 50 50 50 87 87 87 T T T 20 400-419 50 50 50 86 86 86 T T T 21 420-439 49 49 49 85 85 85 F F F 22 440-459 48 48 48 80 80 80 F F F 23 460-479 50 50 50 87 87 87 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 49 49 49 85 85 85 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 50, Max flush: 87 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 87 87 87 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 82 82 82 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 51 51 51 88 88 88 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 50 50 50 87 87 87 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 46 46 46 77 77 77 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 48 48 48 82 82 82 F F F 14 280-299 48 48 48 82 82 82 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 51 51 51 88 88 88 F F F 20 400-419 51 51 51 88 88 88 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 48 48 48 80 80 80 F F F 23 460-479 50 50 50 87 87 87 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 49 49 49 85 85 85 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 50 50 50 87 87 87 F F F 29 580-599 41 41 41 68 68 68 F F F Max gen: 51, Max flush: 88 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.146895879Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:56.200449848Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.200890309Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.23851635Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.239221122Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.23926422Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.239281706Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.239299312Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"Crucible 15b72f66-ccaf-4193-9c4d-2f3f6a83e82d has session id: f2cc36eb-42ef-47e4-94d5-e3aef93652fa","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.24026627Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Upstairs opts: Upstairs UUID: 15b72f66-ccaf-4193-9c4d-2f3f6a83e82d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.240414664Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Crucible stats registered with UUID: 15b72f66-ccaf-4193-9c4d-2f3f6a83e82d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.240439795Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:56.240455227Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241242608Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241271868Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"15b72f66-ccaf-4193-9c4d-2f3f6a83e82d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.24137572Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241403778Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241433589Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241450373Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.24146831Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241484312Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241501247Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241517601Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241676325Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241719223Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:33886)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.241973403Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:33753)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.242038136Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:58955)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.242058187Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.242940592Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.242967788Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.243435895Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.243725287Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.243978616Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.244349004Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.244872996Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.24493394Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[0]R flush_numbers[0..12]: [87, 82, 82, 77, 77, 77, 88, 84, 87, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245154942Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[0]R generation[0..12]: [50, 48, 48, 46, 46, 46, 51, 48, 50, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245181557Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245200175Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[1]R flush_numbers[0..12]: [87, 82, 82, 77, 77, 77, 88, 84, 87, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245217591Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[1]R generation[0..12]: [50, 48, 48, 46, 46, 46, 51, 48, 50, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245234726Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245252472Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[2]R flush_numbers[0..12]: [87, 82, 82, 77, 77, 77, 88, 84, 87, 85, 77, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245269627Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[2]R generation[0..12]: [50, 48, 48, 46, 46, 46, 51, 48, 50, 49, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245286602Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245303677Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245325131Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245341394Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245357286Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245373119Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"":"downstairs","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245389162Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245403852Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"0","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245420315Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"1","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245436599Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"client":"2","":"downstairs","session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245453133Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245468183Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"15b72f66-ccaf-4193-9c4d-2f3f6a83e82d is now active with session: f2cc36eb-42ef-47e4-94d5-e3aef93652fa","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245484366Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245499036Z","hostname":"bmat-EVT22200007-0000589c","pid":1395,"session_id":"f2cc36eb-42ef-47e4-94d5-e3aef93652fa"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245514117Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} {"msg":"Activated sub_volume 15b72f66-ccaf-4193-9c4d-2f3f6a83e82d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:04:56.245527775Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 16384 data: 5 4 5 4 02/30 Read block 479 len 36864 03/30 Read block 459 len 8192 04/30 Read block 377 len 20480 05/30 Write block 37 len 40960 data: 4 3 4 6 5 6 7 6 5 7 06/30 Read block 125 len 20480 07/30 Write block 386 len 28672 data: 6 9 8 8 8 9 8 08/30 Write block 133 len 24576 data: 4 3 3 4 5 6 09/30 Read block 367 len 4096 10/30 Read block 70 len 4096 11/30 Read block 246 len 20480 12/30 Write block 124 len 4096 data: 3 13/30 Read block 582 len 28672 14/30 Write block 289 len 24576 data: 4 4 4 4 4 5 15/30 Write block 175 len 20480 data: 5 5 4 6 6 16/30 Write block 457 len 40960 data: 5 5 5 6 7 8 9 10 10 8 17/30 Read block 87 len 16384 18/30 Read block 465 len 8192 19/30 Read block 497 len 16384 20/30 Read block 443 len 32768 21/30 Read block 414 len 24576 22/30 Write block 204 len 36864 data: 4 3 4 3 3 4 5 3 3 23/30 Read block 510 len 24576 24/30 Write block 574 len 40960 data: 5 5 6 4 4 4 4 4 4 5 25/30 Flush 26/30 Write block 401 len 12288 data: 6 6 7 27/30 Write block 265 len 36864 data: 6 8 7 7 6 6 5 4 5 28/30 Read block 117 len 20480 29/30 Write block 292 len 28672 data: 5 5 6 7 8 9 8 30/30 Read block 35 len 36864 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:04:56.274598933Z","hostname":"bmat-EVT22200007-0000589c","pid":1395} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 87 87 87 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 89 89 89 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 52 52 52 89 89 89 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 52 52 52 89 89 89 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 52 52 52 89 89 89 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 48 48 48 82 82 82 T T T 14 280-299 52 52 52 89 89 89 T T T 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 52 52 52 89 89 89 F F F 20 400-419 51 51 51 88 88 88 T T T 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 52 52 52 89 89 89 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 49 49 49 85 85 85 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 52 52 52 89 89 89 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 52, Max flush: 89 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 87 87 87 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 89 89 89 F F F 3 060-079 46 46 46 77 77 77 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 46 46 46 77 77 77 F F F 6 120-139 52 52 52 89 89 89 F F F 7 140-159 48 48 48 84 84 84 F F F 8 160-179 52 52 52 89 89 89 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 52 52 52 89 89 89 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 53 53 53 90 90 90 F F F 14 280-299 53 53 53 90 90 90 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 84 84 84 F F F 17 340-359 48 48 48 84 84 84 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 52 52 52 89 89 89 F F F 20 400-419 53 53 53 90 90 90 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 52 52 52 89 89 89 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 49 49 49 85 85 85 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 48 48 48 82 82 82 F F F 28 560-579 52 52 52 89 89 89 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 53, Max flush: 90 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.577789874Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:00.63042859Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.630916448Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.66845457Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.669650697Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.669694807Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.669713866Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.669732354Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"Crucible 7f5e070b-77ea-4adb-9e6b-b05edb2853d9 has session id: 92322fed-9e8a-48e7-8da3-17f596c7ab17","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671041713Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Upstairs opts: Upstairs UUID: 7f5e070b-77ea-4adb-9e6b-b05edb2853d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671095884Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Crucible stats registered with UUID: 7f5e070b-77ea-4adb-9e6b-b05edb2853d9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671115604Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:00.671131466Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671753279Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671783761Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"7f5e070b-77ea-4adb-9e6b-b05edb2853d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671872242Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671895319Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671925701Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671943097Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671961394Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.671978209Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672221236Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.67249695Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672520649Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672538615Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"ds_connection connected from Ok(127.0.0.1:44098)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672562043Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"ds_connection connected from Ok(127.0.0.1:51492)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672579549Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"ds_connection connected from Ok(127.0.0.1:49013)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.672807235Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.673302909Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.673432544Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.673473067Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.674073285Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.674260458Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"0","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.674394903Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.674838801Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675149988Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[0]R flush_numbers[0..12]: [87, 89, 89, 77, 77, 77, 89, 84, 89, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675326229Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[0]R generation[0..12]: [50, 52, 52, 46, 46, 46, 52, 48, 52, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675349356Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675367994Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[1]R flush_numbers[0..12]: [87, 89, 89, 77, 77, 77, 89, 84, 89, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675391662Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[1]R generation[0..12]: [50, 52, 52, 46, 46, 46, 52, 48, 52, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675409529Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675427796Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[2]R flush_numbers[0..12]: [87, 89, 89, 77, 77, 77, 89, 84, 89, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675444811Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[2]R generation[0..12]: [50, 52, 52, 46, 46, 46, 52, 48, 52, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675462077Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675479673Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675496096Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.67551281Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675529154Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675545658Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"":"downstairs","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675562302Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675577493Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.67561031Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"1","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675627295Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"client":"2","":"downstairs","session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675644109Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675662787Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"7f5e070b-77ea-4adb-9e6b-b05edb2853d9 is now active with session: 92322fed-9e8a-48e7-8da3-17f596c7ab17","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675679421Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675694522Z","hostname":"bmat-EVT22200007-0000589c","pid":1403,"session_id":"92322fed-9e8a-48e7-8da3-17f596c7ab17"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675709904Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} {"msg":"Activated sub_volume 7f5e070b-77ea-4adb-9e6b-b05edb2853d9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:00.675724393Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 36864 data: 5 6 6 5 4 3 3 3 3 02/30 Read block 373 len 8192 03/30 Write block 116 len 40960 data: 8 7 6 5 5 3 3 2 4 3 04/30 Flush 05/30 Write block 134 len 24576 data: 4 4 5 6 7 7 06/30 Write block 549 len 32768 data: 6 5 4 4 4 4 3 5 07/30 Write block 314 len 32768 data: 4 5 6 7 7 6 7 5 08/30 Write block 61 len 40960 data: 6 5 5 4 4 3 4 4 4 3 09/30 Read block 362 len 28672 10/30 Write block 358 len 36864 data: 5 5 5 4 4 5 6 8 9 11/30 Flush 12/30 Write block 342 len 16384 data: 5 5 6 3 13/30 Write block 341 len 32768 data: 4 6 6 7 4 3 3 3 14/30 Write block 156 len 40960 data: 3 3 5 9 10 9 8 8 8 6 15/30 Read block 361 len 40960 16/30 Write block 310 len 4096 data: 3 17/30 Write block 143 len 4096 data: 10 18/30 Write block 595 len 12288 data: 3 3 3 19/30 Write block 508 len 20480 data: 5 4 4 4 4 20/30 Read block 139 len 8192 21/30 Write block 12 len 12288 data: 3 2 3 22/30 Write block 404 len 36864 data: 8 7 6 6 8 8 8 8 9 23/30 Write block 356 len 16384 data: 5 6 6 6 24/30 Write block 149 len 32768 data: 5 4 4 3 2 2 3 4 25/30 Read block 355 len 24576 26/30 Write block 475 len 12288 data: 2 3 4 27/30 Write block 551 len 24576 data: 5 5 5 5 4 6 28/30 Write block 395 len 8192 data: 7 7 29/30 Write block 170 len 24576 data: 6 7 9 8 7 6 30/30 Write block 146 len 24576 data: 9 6 6 6 5 5 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 17 17 18 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:00.701364361Z","hostname":"bmat-EVT22200007-0000589c","pid":1403} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 87 87 87 T T T 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 89 89 89 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 54 54 54 91 91 91 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 48 48 48 84 84 84 T T T 8 160-179 52 52 52 89 89 89 T T T 9 180-199 49 49 49 85 85 85 F F F 10 200-219 52 52 52 89 89 89 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 53 53 53 90 90 90 F F F 14 280-299 53 53 53 90 90 90 F F F 15 300-319 54 54 54 92 92 92 T T T 16 320-339 54 54 54 92 92 92 F F F 17 340-359 54 54 54 92 92 92 T T T 18 360-379 54 54 54 92 92 92 F F F 19 380-399 52 52 52 89 89 89 T T T 20 400-419 53 53 53 90 90 90 T T T 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 52 52 52 89 89 89 T T T 24 480-499 50 50 50 87 87 87 F F F 25 500-519 49 49 49 85 85 85 T T T 26 520-539 47 47 47 78 78 78 F F F 27 540-559 54 54 54 92 92 92 T T T 28 560-579 52 52 52 89 89 89 F F F 29 580-599 52 52 52 89 89 89 T T T Max gen: 54, Max flush: 92 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 93 93 93 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 89 89 89 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 46 46 46 77 77 77 F F F 5 100-119 54 54 54 91 91 91 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 55 55 55 93 93 93 F F F 8 160-179 55 55 55 93 93 93 F F F 9 180-199 49 49 49 85 85 85 F F F 10 200-219 52 52 52 89 89 89 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 53 53 53 90 90 90 F F F 14 280-299 53 53 53 90 90 90 F F F 15 300-319 55 55 55 93 93 93 F F F 16 320-339 54 54 54 92 92 92 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 54 54 54 92 92 92 F F F 19 380-399 55 55 55 93 93 93 F F F 20 400-419 55 55 55 93 93 93 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 55 55 55 93 93 93 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 55 55 55 93 93 93 F F F 28 560-579 52 52 52 89 89 89 F F F 29 580-599 55 55 55 93 93 93 F F F Max gen: 55, Max flush: 93 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.103196506Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.158874621Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.159655328Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.159771736Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.159804564Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.159837521Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"Crucible 4edc73b1-f43e-474a-84bd-c7e8778b7198 has session id: 7f86bbd4-34de-4ce4-bbac-7189126d9445","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.161237626Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Upstairs opts: Upstairs UUID: 4edc73b1-f43e-474a-84bd-c7e8778b7198, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.161327129Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Crucible stats registered with UUID: 4edc73b1-f43e-474a-84bd-c7e8778b7198","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16136207Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:05.161390899Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162199473Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162254917Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"4edc73b1-f43e-474a-84bd-c7e8778b7198 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162291422Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162309058Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16234919Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16236882Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162387418Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.162937654Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163169598Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163210051Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163229541Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163247538Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"ds_connection connected from Ok(127.0.0.1:33588)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163265444Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"ds_connection connected from Ok(127.0.0.1:64768)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163283501Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"ds_connection connected from Ok(127.0.0.1:41733)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163301198Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163802523Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163840531Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.163870202Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.164748519Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.165028962Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.165056369Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.165409822Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166053879Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[0]R flush_numbers[0..12]: [93, 89, 89, 92, 77, 91, 92, 93, 93, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166244008Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[0]R generation[0..12]: [55, 52, 52, 54, 46, 54, 54, 55, 55, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166273088Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166292357Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[1]R flush_numbers[0..12]: [93, 89, 89, 92, 77, 91, 92, 93, 93, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166310294Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[1]R generation[0..12]: [55, 52, 52, 54, 46, 54, 54, 55, 55, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16632772Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166346308Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[2]R flush_numbers[0..12]: [93, 89, 89, 92, 77, 91, 92, 93, 93, 85, 89, 75]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166364094Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[2]R generation[0..12]: [55, 52, 52, 54, 46, 54, 54, 55, 55, 49, 52, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16638155Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166399466Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16642578Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:05:05.166554143Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166587712Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166604777Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"":"downstairs","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166621581Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166637002Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"0","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:05:05.166908067Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"1","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166934341Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"client":"2","":"downstairs","session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166952237Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.16696823Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"4edc73b1-f43e-474a-84bd-c7e8778b7198 is now active with session: 7f86bbd4-34de-4ce4-bbac-7189126d9445","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.166985215Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.167000677Z","hostname":"bmat-EVT22200007-0000589c","pid":1411,"session_id":"7f86bbd4-34de-4ce4-bbac-7189126d9445"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.167016449Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} {"msg":"Activated sub_volume 4edc73b1-f43e-474a-84bd-c7e8778b7198","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:05.167031139Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 167 len 4096 data: 8 03/30 Flush 04/30 Write block 218 len 32768 data: 5 6 6 6 8 7 7 7 05/30 Write block 13 len 8192 data: 3 4 06/30 Write block 162 len 20480 data: 9 9 9 7 6 07/30 Write block 242 len 16384 data: 6 6 6 6 08/30 Write block 462 len 8192 data: 9 10 09/30 Read block 87 len 4096 10/30 Write block 571 len 12288 data: 6 6 5 11/30 Flush 12/30 Write block 78 len 32768 data: 7 7 7 5 6 4 5 5 13/30 Read block 298 len 20480 14/30 Read block 356 len 16384 15/30 Write block 96 len 16384 data: 6 6 5 7 16/30 Read block 178 len 36864 17/30 Flush 18/30 Write block 238 len 4096 data: 5 19/30 Write block 293 len 36864 data: 6 7 8 9 10 9 7 7 6 20/30 Write block 173 len 36864 data: 9 8 7 6 5 7 7 4 3 21/30 Write block 330 len 16384 data: 5 6 6 5 22/30 Read block 192 len 4096 23/30 Write block 583 len 8192 data: 6 5 24/30 Write block 145 len 28672 data: 12 10 7 7 7 6 6 25/30 Write block 476 len 12288 data: 4 5 5 26/30 Read block 153 len 12288 27/30 Write block 52 len 40960 data: 6 6 5 4 4 6 5 5 7 7 28/30 Read block 518 len 12288 29/30 Read block 316 len 40960 30/30 Read block 116 len 28672 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 13 12 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:05.192011838Z","hostname":"bmat-EVT22200007-0000589c","pid":1411} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 89 89 89 T T T 3 060-079 56 56 56 97 97 97 T T T 4 080-099 56 56 56 97 97 97 F F F 5 100-119 54 54 54 91 91 91 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 55 55 55 93 93 93 T T T 8 160-179 56 56 56 96 96 96 T T T 9 180-199 49 49 49 85 85 85 T T T 10 200-219 56 56 56 96 96 96 F F F 11 220-239 56 56 56 96 96 96 T T T 12 240-259 56 56 56 96 96 96 F F F 13 260-279 53 53 53 90 90 90 F F F 14 280-299 53 53 53 90 90 90 T T T 15 300-319 55 55 55 93 93 93 T T T 16 320-339 54 54 54 92 92 92 T T T 17 340-359 55 55 55 93 93 93 F F F 18 360-379 54 54 54 92 92 92 F F F 19 380-399 55 55 55 93 93 93 F F F 20 400-419 55 55 55 93 93 93 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 56 56 56 96 96 96 T T T 24 480-499 50 50 50 87 87 87 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 55 55 55 93 93 93 F F F 28 560-579 56 56 56 96 96 96 F F F 29 580-599 55 55 55 93 93 93 T T T Max gen: 56, Max flush: 97 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 57 57 57 98 98 98 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 54 54 54 91 91 91 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 57 57 57 98 98 98 F F F 8 160-179 57 57 57 98 98 98 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 57 57 57 98 98 98 F F F 12 240-259 56 56 56 96 96 96 F F F 13 260-279 53 53 53 90 90 90 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 57 57 57 98 98 98 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 54 54 54 92 92 92 F F F 19 380-399 55 55 55 93 93 93 F F F 20 400-419 55 55 55 93 93 93 F F F 21 420-439 49 49 49 85 85 85 F F F 22 440-459 52 52 52 89 89 89 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 55 55 55 93 93 93 F F F 28 560-579 56 56 56 96 96 96 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 57, Max flush: 98 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.561584342Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.615386829Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.616177376Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.616266308Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.616293644Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.616321451Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"Crucible 57c7d08e-5823-4f21-af95-b725efff104e has session id: 7e3db7d6-49e4-4c3d-a792-5483bd2cd395","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.617388124Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Upstairs opts: Upstairs UUID: 57c7d08e-5823-4f21-af95-b725efff104e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.617455491Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Crucible stats registered with UUID: 57c7d08e-5823-4f21-af95-b725efff104e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.61749861Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:09.617527779Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618120762Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618154031Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"57c7d08e-5823-4f21-af95-b725efff104e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618274988Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.61829574Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618328217Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618344881Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618362236Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618377929Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618523416Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618716702Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618931611Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618952664Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"ds_connection connected from Ok(127.0.0.1:53427)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618970271Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"ds_connection connected from Ok(127.0.0.1:48844)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.618993989Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"ds_connection connected from Ok(127.0.0.1:42337)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.619011094Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.619937028Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.619967831Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.619984876Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.620933387Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.620971315Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.621062982Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.621821103Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.6219417Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[0]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 91, 92, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622101557Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[0]R generation[0..12]: [56, 52, 57, 57, 56, 54, 54, 57, 57, 57, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622125416Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622142691Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[1]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 91, 92, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622159365Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[1]R generation[0..12]: [56, 52, 57, 57, 56, 54, 54, 57, 57, 57, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.62218121Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622198525Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[2]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 91, 92, 98, 98, 98, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622214979Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[2]R generation[0..12]: [56, 52, 57, 57, 56, 54, 54, 57, 57, 57, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622230741Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622247215Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622262757Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622278389Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.62229368Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.62230856Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"":"downstairs","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622323882Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622338051Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"0","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622353943Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"1","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622369946Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"client":"2","":"downstairs","session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622385799Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622400098Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"57c7d08e-5823-4f21-af95-b725efff104e is now active with session: 7e3db7d6-49e4-4c3d-a792-5483bd2cd395","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622418926Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622433416Z","hostname":"bmat-EVT22200007-0000589c","pid":1419,"session_id":"7e3db7d6-49e4-4c3d-a792-5483bd2cd395"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622447805Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} {"msg":"Activated sub_volume 57c7d08e-5823-4f21-af95-b725efff104e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:09.622471083Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 32768 data: 11 9 8 5 4 4 2 2 02/30 Write block 547 len 12288 data: 7 7 7 03/30 Read block 110 len 24576 04/30 Read block 570 len 28672 05/30 Flush 06/30 Write block 233 len 24576 data: 5 4 4 4 5 6 07/30 Flush 08/30 Write block 245 len 12288 data: 7 6 5 09/30 Write block 129 len 8192 data: 3 4 10/30 Read block 246 len 40960 11/30 Write block 352 len 40960 data: 4 4 4 4 6 7 7 7 6 5 12/30 Write block 262 len 8192 data: 6 6 13/30 Read block 518 len 12288 14/30 Write block 568 len 16384 data: 7 5 7 7 15/30 Read block 471 len 28672 16/30 Read block 270 len 20480 17/30 Write block 374 len 32768 data: 12 10 9 6 5 5 3 3 18/30 Flush 19/30 Flush 20/30 Write block 449 len 32768 data: 4 4 3 3 3 2 3 5 21/30 Read block 408 len 12288 22/30 Write block 340 len 24576 data: 7 5 7 7 8 5 23/30 Read block 448 len 36864 24/30 Read block 102 len 16384 25/30 Read block 81 len 12288 26/30 Write block 232 len 8192 data: 5 6 27/30 Write block 415 len 36864 data: 5 3 3 4 5 7 5 5 5 28/30 Write block 247 len 28672 data: 6 5 5 6 6 5 4 29/30 Read block 525 len 36864 30/30 Write block 108 len 24576 data: 7 5 5 5 6 8 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:09.650289729Z","hostname":"bmat-EVT22200007-0000589c","pid":1419} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 57 57 57 98 98 98 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 54 54 54 91 91 91 T T T 6 120-139 58 58 58 101 101 101 F F F 7 140-159 57 57 57 98 98 98 F F F 8 160-179 57 57 57 98 98 98 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 58 58 58 100 100 100 T T T 12 240-259 58 58 58 101 101 101 T T T 13 260-279 58 58 58 101 101 101 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 57 57 57 98 98 98 F F F 17 340-359 58 58 58 101 101 101 T T T 18 360-379 58 58 58 101 101 101 F F F 19 380-399 58 58 58 101 101 101 F F F 20 400-419 55 55 55 93 93 93 T T T 21 420-439 49 49 49 85 85 85 T T T 22 440-459 52 52 52 89 89 89 T T T 23 460-479 57 57 57 98 98 98 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 58 58 58 99 99 99 F F F 28 560-579 58 58 58 101 101 101 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 58, Max flush: 101 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 57 57 57 98 98 98 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 58 58 58 101 101 101 F F F 7 140-159 57 57 57 98 98 98 F F F 8 160-179 57 57 57 98 98 98 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 59 59 59 102 102 102 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 57 57 57 98 98 98 F F F 17 340-359 59 59 59 102 102 102 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 58 58 58 101 101 101 F F F 20 400-419 59 59 59 102 102 102 F F F 21 420-439 59 59 59 102 102 102 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 50 50 50 87 87 87 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 58 58 58 99 99 99 F F F 28 560-579 58 58 58 101 101 101 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 59, Max flush: 102 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:13.985351575Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.039301294Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.040030857Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.040090468Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.040127414Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.040161203Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"Crucible 2a392efa-3fbc-4748-b1c5-3dc5b24d54eb has session id: 6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.041171951Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Upstairs opts: Upstairs UUID: 2a392efa-3fbc-4748-b1c5-3dc5b24d54eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.041309994Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Crucible stats registered with UUID: 2a392efa-3fbc-4748-b1c5-3dc5b24d54eb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.041342971Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:14.041371129Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042175615Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042212881Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"2a392efa-3fbc-4748-b1c5-3dc5b24d54eb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.04240799Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042442901Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042748066Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042896429Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042922102Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042947514Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.042967064Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.04323904Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.043334496Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.043368676Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54657)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.04340009Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40212)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.043429249Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36551)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.043457858Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.044148661Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.044355284Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.04456359Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.045124458Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.045286369Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.045696228Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.045934345Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046063669Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[0]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 102, 101, 98, 98, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046483418Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[0]R generation[0..12]: [56, 52, 57, 57, 56, 59, 58, 57, 57, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046549483Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046571448Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[1]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 102, 101, 98, 98, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046588864Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[1]R generation[0..12]: [56, 52, 57, 57, 56, 59, 58, 57, 57, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046605458Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046622352Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[2]R flush_numbers[0..12]: [96, 89, 98, 98, 97, 102, 101, 98, 98, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046641542Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[2]R generation[0..12]: [56, 52, 57, 57, 56, 59, 58, 57, 57, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046657554Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046674339Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046690011Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046705693Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046721255Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046736566Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"":"downstairs","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046758321Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046773191Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"0","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046789735Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"1","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046805156Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"client":"2","":"downstairs","session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046820788Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046835529Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"2a392efa-3fbc-4748-b1c5-3dc5b24d54eb is now active with session: 6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046850569Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046864788Z","hostname":"bmat-EVT22200007-0000589c","pid":1427,"session_id":"6b71c7a9-c5f7-4108-b8a6-8cc67d50cde5"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046879258Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} {"msg":"Activated sub_volume 2a392efa-3fbc-4748-b1c5-3dc5b24d54eb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:14.046930653Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 36864 data: 6 7 9 8 8 7 7 6 5 02/30 Write block 241 len 36864 data: 5 7 7 7 8 7 7 6 6 03/30 Read block 80 len 28672 04/30 Read block 491 len 12288 05/30 Read block 86 len 32768 06/30 Read block 123 len 8192 07/30 Write block 556 len 40960 data: 7 7 6 6 4 6 5 3 4 4 08/30 Write block 396 len 24576 data: 8 5 6 6 8 7 09/30 Read block 279 len 8192 10/30 Read block 411 len 32768 11/30 Read block 452 len 8192 12/30 Write block 134 len 12288 data: 5 5 6 13/30 Read block 303 len 12288 14/30 Read block 484 len 40960 15/30 Read block 23 len 32768 16/30 Read block 281 len 40960 17/30 Write block 553 len 8192 data: 6 6 18/30 Flush 19/30 Write block 345 len 24576 data: 6 4 4 4 2 3 20/30 Write block 168 len 4096 data: 8 21/30 Write block 483 len 4096 data: 6 22/30 Flush 23/30 Write block 38 len 24576 data: 4 5 7 6 7 8 24/30 Write block 124 len 24576 data: 5 4 4 4 3 4 25/30 Read block 295 len 12288 26/30 Read block 225 len 12288 27/30 Read block 548 len 8192 28/30 Write block 167 len 20480 data: 9 9 7 7 8 29/30 Read block 150 len 40960 30/30 Write block 391 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:14.073512308Z","hostname":"bmat-EVT22200007-0000589c","pid":1427} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 52 52 52 89 89 89 T T T 2 040-059 57 57 57 98 98 98 T T T 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 60 60 60 103 103 103 T T T 7 140-159 57 57 57 98 98 98 F F F 8 160-179 60 60 60 104 104 104 T T T 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 57 57 57 98 98 98 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 60 60 60 103 103 103 T T T 20 400-419 60 60 60 103 103 103 F F F 21 420-439 59 59 59 102 102 102 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 60, Max flush: 104 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 61 61 61 105 105 105 F F F 2 040-059 61 61 61 105 105 105 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 61 61 61 105 105 105 F F F 7 140-159 57 57 57 98 98 98 F F F 8 160-179 61 61 61 105 105 105 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 57 57 57 98 98 98 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 58 58 58 101 101 101 F F F 19 380-399 61 61 61 105 105 105 F F F 20 400-419 60 60 60 103 103 103 F F F 21 420-439 59 59 59 102 102 102 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 55 55 55 93 93 93 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 61, Max flush: 105 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.40794632Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.462852004Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.464511109Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.464564658Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.464584128Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.464602956Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"Crucible c9e984d7-bd77-4802-891f-aa695db08efb has session id: 7deb8e25-0c73-4775-b48e-10fc2b95702d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.465478989Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Upstairs opts: Upstairs UUID: c9e984d7-bd77-4802-891f-aa695db08efb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.465527879Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Crucible stats registered with UUID: c9e984d7-bd77-4802-891f-aa695db08efb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.465547369Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:18.465563221Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.466490328Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.466520359Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"c9e984d7-bd77-4802-891f-aa695db08efb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.466612448Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.46663795Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467096969Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467365038Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467389678Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467406853Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.46742487Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467441614Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.46745892Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467636433Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50481)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467666424Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59670)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.467686285Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63048)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.46770357Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.468032303Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.468336585Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.468401949Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.469040165Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.469082231Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.469245595Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.46980521Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470015009Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[0]R flush_numbers[0..12]: [96, 105, 105, 98, 97, 102, 105, 98, 105, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470144484Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[0]R generation[0..12]: [56, 61, 61, 57, 56, 59, 61, 57, 61, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470172261Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470190769Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[1]R flush_numbers[0..12]: [96, 105, 105, 98, 97, 102, 105, 98, 105, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470207593Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[1]R generation[0..12]: [56, 61, 61, 57, 56, 59, 61, 57, 61, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470224157Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470241132Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[2]R flush_numbers[0..12]: [96, 105, 105, 98, 97, 102, 105, 98, 105, 98, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470257646Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[2]R generation[0..12]: [56, 61, 61, 57, 56, 59, 61, 57, 61, 57, 56, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470273418Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470289842Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470312418Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470328461Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470343411Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470417403Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"":"downstairs","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470439037Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470453867Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"0","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.47046993Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"1","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470485893Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"client":"2","":"downstairs","session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:05:18.470501475Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470609415Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"c9e984d7-bd77-4802-891f-aa695db08efb is now active with session: 7deb8e25-0c73-4775-b48e-10fc2b95702d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470834817Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470855609Z","hostname":"bmat-EVT22200007-0000589c","pid":1435,"session_id":"7deb8e25-0c73-4775-b48e-10fc2b95702d"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470871431Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} {"msg":"Activated sub_volume c9e984d7-bd77-4802-891f-aa695db08efb","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:18.470885661Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 363 len 12288 data: 6 7 9 03/30 Write block 372 len 36864 data: 6 10 13 11 10 7 6 6 4 04/30 Read block 81 len 16384 05/30 Read block 560 len 28672 06/30 Write block 53 len 24576 data: 7 6 5 5 7 6 07/30 Write block 228 len 24576 data: 4 3 3 4 6 7 08/30 Flush 09/30 Write block 323 len 32768 data: 4 5 5 4 6 5 4 6 10/30 Flush 11/30 Write block 391 len 36864 data: 11 9 7 8 8 9 6 7 7 12/30 Read block 485 len 40960 13/30 Write block 372 len 28672 data: 7 11 14 12 11 8 7 14/30 Read block 11 len 40960 15/30 Write block 427 len 32768 data: 4 4 4 4 5 7 8 7 16/30 Flush 17/30 Write block 172 len 24576 data: 10 10 9 8 7 6 18/30 Flush 19/30 Write block 510 len 16384 data: 5 5 5 7 20/30 Read block 382 len 40960 21/30 Read block 423 len 20480 22/30 Read block 22 len 24576 23/30 Read block 147 len 36864 24/30 Read block 222 len 40960 25/30 Read block 321 len 4096 26/30 Write block 135 len 20480 data: 6 7 7 8 8 27/30 Read block 389 len 12288 28/30 Read block 453 len 36864 29/30 Read block 520 len 4096 30/30 Read block 324 len 16384 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:18.499671074Z","hostname":"bmat-EVT22200007-0000589c","pid":1435} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 61 61 61 105 105 105 F F F 2 040-059 62 62 62 107 107 107 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 61 61 61 105 105 105 T T T 7 140-159 57 57 57 98 98 98 F F F 8 160-179 62 62 62 110 110 110 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 62 62 62 109 109 109 F F F 20 400-419 60 60 60 103 103 103 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 55 55 55 93 93 93 T T T 26 520-539 47 47 47 78 78 78 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 62, Max flush: 110 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 F F F 1 020-039 61 61 61 105 105 105 F F F 2 040-059 62 62 62 107 107 107 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 57 57 57 98 98 98 F F F 8 160-179 62 62 62 110 110 110 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 56 56 56 96 96 96 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 62 62 62 109 109 109 F F F 20 400-419 60 60 60 103 103 103 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 63 63 63 111 111 111 F F F 26 520-539 47 47 47 78 78 78 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 57 57 57 98 98 98 F F F Max gen: 63, Max flush: 111 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.79082672Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:22.843570682Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.844031154Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.881336148Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.883066528Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.883183808Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.883215914Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.883247729Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"Crucible 1a6ba033-1222-4b32-8d64-31b118935a1e has session id: 9297b2bb-6d1f-4153-a049-737dafff1925","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.884171529Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Upstairs opts: Upstairs UUID: 1a6ba033-1222-4b32-8d64-31b118935a1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.884222363Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Crucible stats registered with UUID: 1a6ba033-1222-4b32-8d64-31b118935a1e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.884241943Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:22.884257746Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885024124Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885057422Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"1a6ba033-1222-4b32-8d64-31b118935a1e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885146504Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.88516895Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885405424Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885446969Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.88546699Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885484275Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885779028Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885849783Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885870686Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885895757Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"ds_connection connected from Ok(127.0.0.1:56433)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885917301Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"ds_connection connected from Ok(127.0.0.1:56694)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.88593618Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"ds_connection connected from Ok(127.0.0.1:44092)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.885954347Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.886888588Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.886971608Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.886992721Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.887491141Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.887576716Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.888022969Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.888261748Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.888376983Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[0]R flush_numbers[0..12]: [96, 105, 107, 98, 97, 102, 111, 98, 110, 98, 96, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.888934374Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[0]R generation[0..12]: [56, 61, 62, 57, 56, 59, 63, 57, 62, 57, 56, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889007654Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889051243Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[1]R flush_numbers[0..12]: [96, 105, 107, 98, 97, 102, 111, 98, 110, 98, 96, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889082026Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[1]R generation[0..12]: [56, 61, 62, 57, 56, 59, 63, 57, 62, 57, 56, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889111135Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889140305Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[2]R flush_numbers[0..12]: [96, 105, 107, 98, 97, 102, 111, 98, 110, 98, 96, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889168312Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[2]R generation[0..12]: [56, 61, 62, 57, 56, 59, 63, 57, 62, 57, 56, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889195899Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889224788Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889251683Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889278368Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889304531Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889331105Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889372671Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.88939695Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"0","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889425238Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"1","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889451652Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"client":"2","":"downstairs","session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889482826Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889508238Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"1a6ba033-1222-4b32-8d64-31b118935a1e is now active with session: 9297b2bb-6d1f-4153-a049-737dafff1925","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889534231Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889558831Z","hostname":"bmat-EVT22200007-0000589c","pid":1443,"session_id":"9297b2bb-6d1f-4153-a049-737dafff1925"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.889584043Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} {"msg":"Activated sub_volume 1a6ba033-1222-4b32-8d64-31b118935a1e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:22.88960714Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 507 len 12288 data: 5 6 5 02/30 Read block 269 len 20480 03/30 Read block 266 len 40960 04/30 Write block 200 len 36864 data: 5 4 4 5 5 4 5 4 4 05/30 Read block 265 len 16384 06/30 Read block 367 len 8192 07/30 Write block 154 len 28672 data: 3 4 5 4 6 10 11 08/30 Read block 467 len 28672 09/30 Read block 204 len 28672 10/30 Write block 47 len 4096 data: 6 11/30 Write block 173 len 12288 data: 11 10 9 12/30 Read block 565 len 32768 13/30 Read block 15 len 28672 14/30 Read block 272 len 36864 15/30 Write block 378 len 16384 data: 8 7 5 4 16/30 Flush 17/30 Write block 413 len 20480 data: 8 6 6 4 4 18/30 Read block 4 len 12288 19/30 Read block 127 len 16384 20/30 Write block 379 len 32768 data: 8 6 5 3 4 4 6 7 21/30 Read block 213 len 40960 22/30 Flush 23/30 Write block 372 len 40960 data: 8 12 15 13 12 9 9 9 7 6 24/30 Write block 12 len 16384 data: 4 4 5 3 25/30 Write block 595 len 16384 data: 4 4 4 2 26/30 Write block 524 len 36864 data: 5 5 4 4 5 6 5 5 4 27/30 Read block 180 len 32768 28/30 Read block 69 len 40960 29/30 Read block 501 len 8192 30/30 Read block 36 len 8192 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:22.919631669Z","hostname":"bmat-EVT22200007-0000589c","pid":1443} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 96 96 96 T T T 1 020-039 61 61 61 105 105 105 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 64 64 64 112 112 112 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 64 64 64 112 112 112 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 64 64 64 113 113 113 T T T 19 380-399 64 64 64 113 113 113 T T T 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 64 64 64 112 112 112 F F F 26 520-539 47 47 47 78 78 78 T T T 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 57 57 57 98 98 98 T T T Max gen: 64, Max flush: 113 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 61 61 61 105 105 105 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 57 57 57 98 98 98 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 64 64 64 112 112 112 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 64 64 64 112 112 112 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 60 60 60 103 103 103 F F F 13 260-279 60 60 60 103 103 103 F F F 14 280-299 57 57 57 98 98 98 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 65 65 65 114 114 114 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 59 59 59 102 102 102 F F F 23 460-479 57 57 57 98 98 98 F F F 24 480-499 60 60 60 104 104 104 F F F 25 500-519 64 64 64 112 112 112 F F F 26 520-539 65 65 65 114 114 114 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 65 65 65 114 114 114 F F F Max gen: 65, Max flush: 114 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.245848609Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.302589518Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.303515793Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.303572259Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.303591688Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.303609054Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"Crucible 9105b752-99fd-45b5-aed3-2c3808b04de2 has session id: 1393710c-fcb4-45e1-ada1-d24cf533d2bf","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.304703473Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Upstairs opts: Upstairs UUID: 9105b752-99fd-45b5-aed3-2c3808b04de2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.304817286Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Crucible stats registered with UUID: 9105b752-99fd-45b5-aed3-2c3808b04de2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.304840864Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:27.304856556Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.305678608Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.305914501Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"9105b752-99fd-45b5-aed3-2c3808b04de2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.30594833Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.305968Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306058776Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306087795Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306124971Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306154592Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306184243Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306333057Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306595214Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.30663237Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46171)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306663714Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:34433)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306694416Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:40973)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306723967Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.306800023Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.307237227Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.307843568Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.308083869Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.308707385Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.308986035Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.309011096Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.309440455Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[0]R flush_numbers[0..12]: [114, 105, 112, 98, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310219248Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[0]R generation[0..12]: [65, 61, 64, 57, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310241564Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310258569Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[1]R flush_numbers[0..12]: [114, 105, 112, 98, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310276074Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[1]R generation[0..12]: [65, 61, 64, 57, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310292899Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310309864Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[2]R flush_numbers[0..12]: [114, 105, 112, 98, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310327249Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[2]R generation[0..12]: [65, 61, 64, 57, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310343432Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310360297Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310375989Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310392012Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310407253Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310445902Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"":"downstairs","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310463418Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310477517Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"0","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.31049367Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"1","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310509502Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"client":"2","":"downstairs","session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310525795Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310540145Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"9105b752-99fd-45b5-aed3-2c3808b04de2 is now active with session: 1393710c-fcb4-45e1-ada1-d24cf533d2bf","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310555636Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310569986Z","hostname":"bmat-EVT22200007-0000589c","pid":1451,"session_id":"1393710c-fcb4-45e1-ada1-d24cf533d2bf"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310584866Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} {"msg":"Activated sub_volume 9105b752-99fd-45b5-aed3-2c3808b04de2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:27.310598684Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 4096 data: 8 02/30 Write block 409 len 16384 data: 9 9 9 10 03/30 Read block 417 len 16384 04/30 Flush 05/30 Write block 597 len 4096 data: 5 06/30 Write block 35 len 20480 data: 5 4 5 5 6 07/30 Read block 232 len 8192 08/30 Read block 532 len 28672 09/30 Flush 10/30 Write block 482 len 20480 data: 6 7 4 5 6 11/30 Read block 72 len 24576 12/30 Read block 352 len 8192 13/30 Read block 71 len 36864 14/30 Flush 15/30 Write block 69 len 16384 data: 5 4 3 3 16/30 Write block 250 len 36864 data: 7 7 6 5 4 6 7 7 6 17/30 Write block 294 len 16384 data: 8 9 10 11 18/30 Read block 581 len 40960 19/30 Read block 98 len 36864 20/30 Flush 21/30 Write block 253 len 32768 data: 6 5 7 8 8 7 6 3 22/30 Write block 456 len 28672 data: 6 6 6 6 7 8 10 23/30 Flush 24/30 Write block 506 len 32768 data: 5 6 7 6 6 6 6 8 25/30 Write block 586 len 36864 data: 4 3 3 4 4 5 4 4 5 26/30 Read block 467 len 8192 27/30 Write block 381 len 32768 data: 7 4 5 5 7 8 10 9 28/30 Read block 37 len 20480 29/30 Write block 549 len 24576 data: 8 6 6 6 7 7 30/30 Write block 34 len 12288 data: 6 6 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:27.341303896Z","hostname":"bmat-EVT22200007-0000589c","pid":1451} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 66 66 66 116 116 116 T T T 2 040-059 64 64 64 112 112 112 F F F 3 060-079 66 66 66 118 118 118 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 64 64 64 112 112 112 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 64 64 64 112 112 112 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 66 66 66 119 119 119 F F F 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 65 65 65 114 114 114 T T T 20 400-419 66 66 66 115 115 115 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 64 64 64 112 112 112 T T T 26 520-539 65 65 65 114 114 114 F F F 27 540-559 60 60 60 103 103 103 T T T 28 560-579 60 60 60 103 103 103 F F F 29 580-599 66 66 66 116 116 116 T T T Max gen: 66, Max flush: 119 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 67 67 67 120 120 120 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 66 66 66 118 118 118 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 64 64 64 112 112 112 F F F 9 180-199 57 57 57 98 98 98 F F F 10 200-219 64 64 64 112 112 112 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 66 66 66 119 119 119 F F F 13 260-279 66 66 66 119 119 119 F F F 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 62 62 62 108 108 108 F F F 17 340-359 60 60 60 104 104 104 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 67 67 67 120 120 120 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 62 62 62 109 109 109 F F F 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 65 65 65 114 114 114 F F F 27 540-559 67 67 67 120 120 120 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 67, Max flush: 120 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.674765313Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.729564229Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.73043468Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.730484241Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.730501597Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.730519243Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"Crucible 203a0dbc-2bd5-45d6-98b3-06a800bdd870 has session id: 64ffd468-d2e4-4b7b-852a-341fe571bc08","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.731462242Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Upstairs opts: Upstairs UUID: 203a0dbc-2bd5-45d6-98b3-06a800bdd870, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.731556896Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Crucible stats registered with UUID: 203a0dbc-2bd5-45d6-98b3-06a800bdd870","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.731598361Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:41.731626899Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.732303103Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.732349267Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"203a0dbc-2bd5-45d6-98b3-06a800bdd870 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.732458931Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.732501689Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.732993615Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733083569Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733104823Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733121757Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733140175Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733156549Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733173984Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733190739Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:41978)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733207593Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:33260)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733224498Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:41783)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733241422Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733628905Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.733661171Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.734348487Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.734400053Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.734419913Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.735295264Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.735325045Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.735702037Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[0]R flush_numbers[0..12]: [114, 120, 112, 118, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736490159Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[0]R generation[0..12]: [65, 67, 64, 66, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736513206Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736530722Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[1]R flush_numbers[0..12]: [114, 120, 112, 118, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736548138Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[1]R generation[0..12]: [65, 67, 64, 66, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736564681Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736581867Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[2]R flush_numbers[0..12]: [114, 120, 112, 118, 97, 102, 111, 112, 112, 98, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736599192Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[2]R generation[0..12]: [65, 67, 64, 66, 56, 59, 63, 64, 64, 57, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736623432Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736641218Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736657271Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736673284Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736688765Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736704327Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"":"downstairs","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.73672054Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736735311Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"0","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736751614Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"1","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Transition from WaitQuorum to Active","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736778359Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"client":"2","":"downstairs","session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736795864Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736810494Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"203a0dbc-2bd5-45d6-98b3-06a800bdd870 is now active with session: 64ffd468-d2e4-4b7b-852a-341fe571bc08","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736826036Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736840676Z","hostname":"bmat-EVT22200007-0000589c","pid":1459,"session_id":"64ffd468-d2e4-4b7b-852a-341fe571bc08"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736855486Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} {"msg":"Activated sub_volume 203a0dbc-2bd5-45d6-98b3-06a800bdd870","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:41.736873523Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 28672 data: 12 10 8 9 9 10 7 02/30 Read block 396 len 24576 03/30 Write block 216 len 8192 data: 5 7 04/30 Write block 200 len 32768 data: 6 5 5 6 6 5 6 5 05/30 Flush 06/30 Write block 178 len 4096 data: 8 07/30 Write block 241 len 28672 data: 6 8 8 8 9 8 8 08/30 Flush 09/30 Write block 195 len 16384 data: 6 5 5 6 10/30 Flush 11/30 Write block 355 len 20480 data: 5 7 8 8 8 12/30 Flush 13/30 Flush 14/30 Write block 480 len 16384 data: 6 7 7 8 15/30 Write block 22 len 8192 data: 4 4 16/30 Read block 330 len 32768 17/30 Read block 549 len 32768 18/30 Write block 160 len 40960 data: 12 10 10 10 10 8 7 10 10 8 19/30 Read block 137 len 12288 20/30 Read block 481 len 36864 21/30 Read block 188 len 24576 22/30 Flush 23/30 Write block 334 len 36864 data: 6 5 6 6 6 7 8 6 8 24/30 Flush 25/30 Write block 429 len 20480 data: 5 5 6 8 9 26/30 Write block 530 len 16384 data: 6 6 5 3 27/30 Read block 289 len 24576 28/30 Write block 25 len 4096 data: 5 29/30 Write block 255 len 36864 data: 8 9 9 8 7 4 6 7 7 30/30 Read block 444 len 8192 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:41.764592515Z","hostname":"bmat-EVT22200007-0000589c","pid":1459} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 68 68 68 126 126 126 T T T 2 040-059 64 64 64 112 112 112 F F F 3 060-079 66 66 66 118 118 118 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 68 68 68 122 122 122 T T T 13 260-279 66 66 66 119 119 119 T T T 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 68 68 68 121 121 121 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 62 62 62 109 109 109 T T T 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 68 68 68 126 126 126 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 65 65 65 114 114 114 T T T 27 540-559 67 67 67 120 120 120 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 68, Max flush: 127 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 69 69 69 128 128 128 F F F 2 040-059 64 64 64 112 112 112 F F F 3 060-079 66 66 66 118 118 118 F F F 4 080-099 56 56 56 97 97 97 F F F 5 100-119 59 59 59 102 102 102 F F F 6 120-139 63 63 63 111 111 111 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 69 69 69 128 128 128 F F F 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 68 68 68 121 121 121 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 69 69 69 128 128 128 F F F 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 68 68 68 126 126 126 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 67 67 67 120 120 120 F F F 28 560-579 60 60 60 103 103 103 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 69, Max flush: 128 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.091195142Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.146152462Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.146940074Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.147028525Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.147051853Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.147070801Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"Crucible d22b9b58-594b-4d4c-a519-6266f343eef9 has session id: 443eaaa1-8ca3-405e-b780-bc857c6c376e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.14793021Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Upstairs opts: Upstairs UUID: d22b9b58-594b-4d4c-a519-6266f343eef9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148022007Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Crucible stats registered with UUID: d22b9b58-594b-4d4c-a519-6266f343eef9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148045315Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:46.14806259Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148742922Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148773705Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"d22b9b58-594b-4d4c-a519-6266f343eef9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148916027Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.148941849Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149143652Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.14919155Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149210539Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149227364Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149244899Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149260481Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149330204Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149795596Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51604)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149830648Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38510)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.149848945Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63616)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.14986576Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.150237771Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.150649954Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.150690377Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.151270814Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.151724613Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.152450107Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.152511352Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.152700289Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[0]R flush_numbers[0..12]: [114, 128, 112, 118, 97, 102, 111, 112, 126, 123, 121, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153408819Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[0]R generation[0..12]: [65, 69, 64, 66, 56, 59, 63, 64, 68, 68, 68, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153432788Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.15345861Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[1]R flush_numbers[0..12]: [114, 128, 112, 118, 97, 102, 111, 112, 126, 123, 121, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153476547Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[1]R generation[0..12]: [65, 69, 64, 66, 56, 59, 63, 64, 68, 68, 68, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153493392Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153510306Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[2]R flush_numbers[0..12]: [114, 128, 112, 118, 97, 102, 111, 112, 126, 123, 121, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153527Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[2]R generation[0..12]: [65, 69, 64, 66, 56, 59, 63, 64, 68, 68, 68, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153543224Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153559787Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153575069Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153590601Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153605501Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153620281Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"":"downstairs","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153635743Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153649651Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"0","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153665143Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"1","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153680795Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"client":"2","":"downstairs","session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153699904Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153714554Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"d22b9b58-594b-4d4c-a519-6266f343eef9 is now active with session: 443eaaa1-8ca3-405e-b780-bc857c6c376e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153729545Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153744185Z","hostname":"bmat-EVT22200007-0000589c","pid":1467,"session_id":"443eaaa1-8ca3-405e-b780-bc857c6c376e"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153758714Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} {"msg":"Activated sub_volume d22b9b58-594b-4d4c-a519-6266f343eef9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:46.153772072Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 36864 data: 7 6 6 7 6 6 7 7 6 02/30 Read block 576 len 4096 03/30 Flush 04/30 Write block 265 len 28672 data: 8 10 9 9 8 8 7 05/30 Write block 426 len 36864 data: 3 5 5 6 6 7 9 10 8 06/30 Read block 173 len 4096 07/30 Write block 40 len 32768 data: 8 7 8 9 7 6 8 7 08/30 Read block 307 len 28672 09/30 Read block 77 len 12288 10/30 Write block 509 len 20480 data: 7 7 7 7 9 11/30 Read block 564 len 12288 12/30 Read block 1 len 4096 13/30 Read block 398 len 40960 14/30 Read block 443 len 4096 15/30 Read block 292 len 32768 16/30 Read block 205 len 32768 17/30 Read block 203 len 20480 18/30 Read block 177 len 32768 19/30 Read block 525 len 36864 20/30 Write block 109 len 24576 data: 6 6 6 7 9 7 21/30 Read block 223 len 28672 22/30 Write block 232 len 20480 data: 7 8 5 5 5 23/30 Read block 302 len 12288 24/30 Write block 115 len 28672 data: 8 9 8 7 6 6 4 25/30 Read block 534 len 4096 26/30 Write block 565 len 20480 data: 5 5 7 8 6 27/30 Write block 52 len 40960 data: 7 8 7 6 6 8 7 6 8 8 28/30 Read block 285 len 32768 29/30 Read block 245 len 32768 30/30 Write block 113 len 16384 data: 10 8 9 10 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:46.185231075Z","hostname":"bmat-EVT22200007-0000589c","pid":1467} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 69 69 69 128 128 128 F F F 2 040-059 64 64 64 112 112 112 T T T 3 060-079 66 66 66 118 118 118 T T T 4 080-099 70 70 70 129 129 129 F F F 5 100-119 59 59 59 102 102 102 T T T 6 120-139 63 63 63 111 111 111 T T T 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 62 62 62 107 107 107 T T T 12 240-259 69 69 69 128 128 128 F F F 13 260-279 69 69 69 128 128 128 T T T 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 68 68 68 121 121 121 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 69 69 69 128 128 128 T T T 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 68 68 68 126 126 126 F F F 25 500-519 67 67 67 120 120 120 T T T 26 520-539 69 69 69 128 128 128 F F F 27 540-559 67 67 67 120 120 120 F F F 28 560-579 60 60 60 103 103 103 T T T 29 580-599 67 67 67 120 120 120 F F F Max gen: 70, Max flush: 129 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 69 69 69 128 128 128 F F F 2 040-059 71 71 71 130 130 130 F F F 3 060-079 71 71 71 130 130 130 F F F 4 080-099 70 70 70 129 129 129 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 71 71 71 130 130 130 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 71 71 71 130 130 130 F F F 14 280-299 66 66 66 118 118 118 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 68 68 68 121 121 121 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 66 66 66 119 119 119 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 68 68 68 126 126 126 F F F 25 500-519 71 71 71 130 130 130 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 67 67 67 120 120 120 F F F 28 560-579 71 71 71 130 130 130 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 71, Max flush: 130 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.541126336Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.599412216Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.600194576Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.600255962Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.600276414Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.600294771Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"Crucible 81a7b2d0-2ee2-4736-9393-534ac8033bd1 has session id: 3f1763de-4283-4bad-8eab-9001b5692106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.601323747Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Upstairs opts: Upstairs UUID: 81a7b2d0-2ee2-4736-9393-534ac8033bd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.601367246Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Crucible stats registered with UUID: 81a7b2d0-2ee2-4736-9393-534ac8033bd1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.601385754Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:50.601402959Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602373705Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602441884Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"81a7b2d0-2ee2-4736-9393-534ac8033bd1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602461504Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602478549Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602546047Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602565287Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602583975Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602601711Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.602809025Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60290428Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603173952Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603218112Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"ds_connection connected from Ok(127.0.0.1:61572)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60325125Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"ds_connection connected from Ok(127.0.0.1:48018)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603275971Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"ds_connection connected from Ok(127.0.0.1:48210)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603295841Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603869866Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.603926422Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60432131Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.604592745Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.604810821Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.605392371Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.605537828Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60573398Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[0]R flush_numbers[0..12]: [114, 128, 130, 130, 129, 130, 130, 112, 126, 123, 121, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606648912Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[0]R generation[0..12]: [65, 69, 71, 71, 70, 71, 71, 64, 68, 68, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606692591Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60671179Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[1]R flush_numbers[0..12]: [114, 128, 130, 130, 129, 130, 130, 112, 126, 123, 121, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606729056Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[1]R generation[0..12]: [65, 69, 71, 71, 70, 71, 71, 64, 68, 68, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60674604Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606762755Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[2]R flush_numbers[0..12]: [114, 128, 130, 130, 129, 130, 130, 112, 126, 123, 121, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60677991Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[2]R generation[0..12]: [65, 69, 71, 71, 70, 71, 71, 64, 68, 68, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606802847Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606820152Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606835994Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606852548Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.6068679Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606883031Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"":"downstairs","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606904114Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606918704Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"0","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606934747Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"1","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606950308Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"client":"2","":"downstairs","session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60696584Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.6069803Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"81a7b2d0-2ee2-4736-9393-534ac8033bd1 is now active with session: 3f1763de-4283-4bad-8eab-9001b5692106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.606995401Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.607009469Z","hostname":"bmat-EVT22200007-0000589c","pid":1475,"session_id":"3f1763de-4283-4bad-8eab-9001b5692106"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.60702426Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} {"msg":"Activated sub_volume 81a7b2d0-2ee2-4736-9393-534ac8033bd1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:50.607038379Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 38 len 32768 data: 6 7 9 8 9 10 8 7 02/30 Read block 20 len 24576 03/30 Read block 431 len 16384 04/30 Write block 341 len 36864 data: 7 9 8 9 7 5 5 5 3 05/30 Read block 559 len 32768 06/30 Write block 506 len 24576 data: 6 7 8 8 8 8 07/30 Write block 502 len 28672 data: 5 4 5 5 7 8 9 08/30 Write block 450 len 40960 data: 5 4 4 4 3 4 7 7 7 7 09/30 Read block 364 len 32768 10/30 Read block 399 len 32768 11/30 Flush 12/30 Write block 387 len 40960 data: 11 10 9 9 13 11 9 10 10 11 13/30 Write block 236 len 16384 data: 6 6 7 4 14/30 Write block 85 len 40960 data: 6 5 6 5 6 8 7 7 8 7 15/30 Read block 527 len 12288 16/30 Read block 81 len 24576 17/30 Read block 537 len 8192 18/30 Write block 487 len 8192 data: 6 6 19/30 Read block 24 len 24576 20/30 Write block 489 len 12288 data: 3 3 3 21/30 Write block 234 len 16384 data: 6 6 7 7 22/30 Read block 438 len 32768 23/30 Read block 259 len 40960 24/30 Read block 12 len 36864 25/30 Write block 482 len 36864 data: 8 9 5 6 7 7 7 4 4 26/30 Write block 555 len 28672 data: 5 8 8 7 7 5 7 27/30 Read block 123 len 12288 28/30 Write block 279 len 16384 data: 7 8 7 6 29/30 Write block 493 len 4096 data: 3 30/30 Read block 303 len 32768 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 2 5 Done 16 19 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:50.635033515Z","hostname":"bmat-EVT22200007-0000589c","pid":1475} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 131 131 131 F F F 3 060-079 71 71 71 130 130 130 F F F 4 080-099 70 70 70 129 129 129 T T T 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 71 71 71 130 130 130 T T T 12 240-259 69 69 69 128 128 128 F F F 13 260-279 71 71 71 130 130 130 T T T 14 280-299 66 66 66 118 118 118 T T T 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 72 72 72 131 131 131 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 68 68 68 121 121 121 T T T 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 72 72 72 131 131 131 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 68 68 68 126 126 126 T T T 25 500-519 72 72 72 131 131 131 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 67 67 67 120 120 120 T T T 28 560-579 71 71 71 130 130 130 T T T 29 580-599 67 67 67 120 120 120 F F F Max gen: 72, Max flush: 131 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 114 114 114 F F F 1 020-039 72 72 72 131 131 131 F F F 2 040-059 72 72 72 131 131 131 F F F 3 060-079 71 71 71 130 130 130 F F F 4 080-099 73 73 73 132 132 132 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 68 68 68 123 123 123 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 73 73 73 132 132 132 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 73 73 73 132 132 132 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 72 72 72 131 131 131 F F F 18 360-379 65 65 65 114 114 114 F F F 19 380-399 73 73 73 132 132 132 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 72 72 72 131 131 131 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 73 73 73 132 132 132 F F F 25 500-519 72 72 72 131 131 131 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 73 73 73 132 132 132 F F F 28 560-579 73 73 73 132 132 132 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 73, Max flush: 132 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:54.965996134Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.020915446Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.021864238Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.02195331Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.021983722Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.022003412Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"Crucible b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b has session id: b675610c-8551-4a1f-b359-6e7ddd4dc012","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.022698464Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Upstairs opts: Upstairs UUID: b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.022870226Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Crucible stats registered with UUID: b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.022896119Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:55.022914156Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.023704172Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.023743412Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.023925355Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.0239574Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024011822Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024041612Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024072556Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024100172Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024138801Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024166608Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024195708Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"ds_connection connected from Ok(127.0.0.1:45406)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024224356Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024252153Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"ds_connection connected from Ok(127.0.0.1:39367)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024406038Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"ds_connection connected from Ok(127.0.0.1:39136)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.024486402Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.025239213Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.025278333Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.025450074Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.026150708Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.026334363Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.026360156Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.02688021Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.02718341Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[0]R flush_numbers[0..12]: [114, 131, 131, 130, 132, 130, 130, 112, 126, 123, 121, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027323888Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[0]R generation[0..12]: [65, 72, 72, 71, 73, 71, 71, 64, 68, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.02734933Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027367367Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[1]R flush_numbers[0..12]: [114, 131, 131, 130, 132, 130, 130, 112, 126, 123, 121, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027384903Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[1]R generation[0..12]: [65, 72, 72, 71, 73, 71, 71, 64, 68, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027401847Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027419303Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[2]R flush_numbers[0..12]: [114, 131, 131, 130, 132, 130, 130, 112, 126, 123, 121, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027436819Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[2]R generation[0..12]: [65, 72, 72, 71, 73, 71, 71, 64, 68, 68, 68, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027453894Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027471199Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027487092Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027502914Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027518847Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027534679Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"":"downstairs","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027550582Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027570372Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"0","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027587377Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"1","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.02760372Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"client":"2","":"downstairs","session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027620294Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:05:55.027635145Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b is now active with session: b675610c-8551-4a1f-b359-6e7ddd4dc012","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027664595Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027679546Z","hostname":"bmat-EVT22200007-0000589c","pid":1483,"session_id":"b675610c-8551-4a1f-b359-6e7ddd4dc012"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027694877Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} {"msg":"Activated sub_volume b6bcfe1a-2fbe-46aa-a6ab-d6a0e32bc53b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:05:55.027708485Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 28672 data: 8 9 7 7 8 8 8 02/30 Read block 344 len 24576 03/30 Read block 210 len 16384 04/30 Read block 374 len 36864 05/30 Write block 244 len 32768 data: 9 10 9 9 7 7 8 8 06/30 Read block 512 len 8192 07/30 Write block 180 len 20480 data: 5 4 2 2 3 08/30 Flush 09/30 Write block 2 len 20480 data: 3 4 4 4 4 10/30 Read block 205 len 20480 11/30 Write block 249 len 36864 data: 8 9 9 7 7 6 9 10 10 12/30 Read block 236 len 16384 13/30 Read block 334 len 36864 14/30 Write block 485 len 20480 data: 7 8 8 8 5 15/30 Read block 572 len 36864 16/30 Read block 344 len 16384 17/30 Write block 393 len 4096 data: 10 18/30 Flush 19/30 Write block 30 len 36864 data: 5 5 6 4 7 7 6 6 7 20/30 Read block 484 len 32768 21/30 Read block 299 len 16384 22/30 Write block 444 len 24576 data: 6 5 6 5 5 5 23/30 Read block 184 len 8192 24/30 Write block 374 len 36864 data: 16 14 13 10 10 10 8 8 5 25/30 Write block 5 len 40960 data: 5 5 4 3 4 4 3 5 5 6 26/30 Read block 384 len 24576 27/30 Write block 190 len 32768 data: 7 7 8 6 6 7 6 6 28/30 Read block 411 len 12288 29/30 Read block 421 len 32768 30/30 Write block 64 len 40960 data: 5 5 4 5 5 6 5 4 4 3 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:05:55.058502448Z","hostname":"bmat-EVT22200007-0000589c","pid":1483} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 134 134 134 T T T 1 020-039 72 72 72 131 131 131 T T T 2 040-059 72 72 72 131 131 131 F F F 3 060-079 71 71 71 130 130 130 T T T 4 080-099 73 73 73 132 132 132 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 74 74 74 133 133 133 T T T 10 200-219 68 68 68 121 121 121 F F F 11 220-239 74 74 74 133 133 133 F F F 12 240-259 74 74 74 134 134 134 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 73 73 73 132 132 132 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 72 72 72 131 131 131 F F F 18 360-379 65 65 65 114 114 114 T T T 19 380-399 74 74 74 134 134 134 T T T 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 72 72 72 131 131 131 T T T 23 460-479 66 66 66 119 119 119 F F F 24 480-499 74 74 74 134 134 134 F F F 25 500-519 72 72 72 131 131 131 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 73 73 73 132 132 132 F F F 28 560-579 73 73 73 132 132 132 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 74, Max flush: 134 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 135 135 135 F F F 1 020-039 75 75 75 135 135 135 F F F 2 040-059 72 72 72 131 131 131 F F F 3 060-079 75 75 75 135 135 135 F F F 4 080-099 73 73 73 132 132 132 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 68 68 68 126 126 126 F F F 9 180-199 75 75 75 135 135 135 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 74 74 74 133 133 133 F F F 12 240-259 74 74 74 134 134 134 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 73 73 73 132 132 132 F F F 15 300-319 57 57 57 98 98 98 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 72 72 72 131 131 131 F F F 18 360-379 75 75 75 135 135 135 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 75 75 75 135 135 135 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 74 74 74 134 134 134 F F F 25 500-519 72 72 72 131 131 131 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 73 73 73 132 132 132 F F F 28 560-579 73 73 73 132 132 132 F F F 29 580-599 67 67 67 120 120 120 F F F Max gen: 75, Max flush: 135 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.399694481Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:09.450862699Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.451278961Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.487983627Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.488725544Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.488804616Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.488822854Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.488841452Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"Crucible ed2b72a5-e892-4ef0-ac41-fae8c04ef511 has session id: 1082a060-cc5d-49db-9c14-1c210ceb3802","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.48980852Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Upstairs opts: Upstairs UUID: ed2b72a5-e892-4ef0-ac41-fae8c04ef511, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49006241Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Crucible stats registered with UUID: ed2b72a5-e892-4ef0-ac41-fae8c04ef511","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.490122262Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:09.490160771Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.490802103Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.490832635Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"ed2b72a5-e892-4ef0-ac41-fae8c04ef511 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49096679Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.490989637Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49109855Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49111834Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49113769Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491154895Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491173092Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491189896Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491207703Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491225048Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"ds_connection connected from Ok(127.0.0.1:53918)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491321015Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"ds_connection connected from Ok(127.0.0.1:54720)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491509901Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"ds_connection connected from Ok(127.0.0.1:56324)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.491540373Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49253092Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.492794399Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49282427Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.493024981Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.493280724Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.493626932Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.493847363Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49396785Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[0]R flush_numbers[0..12]: [135, 135, 131, 135, 132, 130, 130, 112, 126, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494520771Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[0]R generation[0..12]: [75, 75, 72, 75, 73, 71, 71, 64, 68, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49454466Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494563308Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[1]R flush_numbers[0..12]: [135, 135, 131, 135, 132, 130, 130, 112, 126, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494581606Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[1]R generation[0..12]: [75, 75, 72, 75, 73, 71, 71, 64, 68, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494598791Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494616337Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[2]R flush_numbers[0..12]: [135, 135, 131, 135, 132, 130, 130, 112, 126, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494638382Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[2]R generation[0..12]: [75, 75, 72, 75, 73, 71, 71, 64, 68, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494655807Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494673403Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494689597Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49470607Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494722173Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494737825Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"":"downstairs","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494753928Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494768498Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"0","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494784541Z",Wait for a query_work_queue command to finish before sending IO "hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"1","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494815314Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"client":"2","":"downstairs","session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494831637Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494846558Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"ed2b72a5-e892-4ef0-ac41-fae8c04ef511 is now active with session: 1082a060-cc5d-49db-9c14-1c210ceb3802","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49486217Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49487678Z","hostname":"bmat-EVT22200007-0000589c","pid":1491,"session_id":"1082a060-cc5d-49db-9c14-1c210ceb3802"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.49489189Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} {"msg":"Activated sub_volume ed2b72a5-e892-4ef0-ac41-fae8c04ef511","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:09.494909066Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 493 len 8192 data: 4 4 02/30 Read block 61 len 36864 03/30 Read block 340 len 8192 04/30 Read block 579 len 36864 05/30 Read block 225 len 12288 06/30 Write block 32 len 16384 data: 7 5 8 8 07/30 Read block 289 len 20480 08/30 Flush 09/30 Write block 498 len 28672 data: 4 5 6 6 6 5 6 10/30 Write block 570 len 20480 data: 8 8 7 6 6 11/30 Read block 371 len 4096 12/30 Write block 302 len 24576 data: 6 5 4 5 6 4 13/30 Write block 342 len 24576 data: 10 9 10 8 6 6 14/30 Read block 475 len 20480 15/30 Flush 16/30 Write block 440 len 40960 data: 6 6 6 5 7 6 7 6 6 6 17/30 Write block 584 len 36864 data: 6 6 5 4 4 5 5 6 5 18/30 Read block 89 len 28672 19/30 Write block 364 len 20480 data: 8 10 10 6 6 20/30 Read block 152 len 20480 21/30 Read block 307 len 24576 22/30 Read block 358 len 40960 23/30 Flush 24/30 Write block 163 len 4096 data: 11 25/30 Read block 36 len 20480 26/30 Flush 27/30 Write block 335 len 32768 data: 6 7 7 7 8 9 8 11 28/30 Write block 66 len 40960 data: 5 6 6 7 6 5 5 4 3 4 29/30 Write block 283 len 4096 data: 4 30/30 Write block 484 len 8192 data: 6 8 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 0 0 0 0 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:09.520729874Z","hostname":"bmat-EVT22200007-0000589c","pid":1491} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 135 135 135 F F F 1 020-039 76 76 76 136 136 136 F F F 2 040-059 72 72 72 131 131 131 F F F 3 060-079 75 75 75 135 135 135 T T T 4 080-099 73 73 73 132 132 132 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 75 75 75 135 135 135 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 74 74 74 133 133 133 F F F 12 240-259 74 74 74 134 134 134 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 73 73 73 132 132 132 T T T 15 300-319 76 76 76 137 137 137 F F F 16 320-339 68 68 68 127 127 127 T T T 17 340-359 76 76 76 137 137 137 T T T 18 360-379 76 76 76 138 138 138 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 76 76 76 138 138 138 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 76 76 76 137 137 137 T T T 25 500-519 76 76 76 137 137 137 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 73 73 73 132 132 132 F F F 28 560-579 76 76 76 137 137 137 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 76, Max flush: 139 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 135 135 135 F F F 1 020-039 76 76 76 136 136 136 F F F 2 040-059 72 72 72 131 131 131 F F F 3 060-079 77 77 77 140 140 140 F F F 4 080-099 73 73 73 132 132 132 F F F 5 100-119 71 71 71 130 130 130 F F F 6 120-139 71 71 71 130 130 130 F F F 7 140-159 64 64 64 112 112 112 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 75 75 75 135 135 135 F F F 10 200-219 68 68 68 121 121 121 F F F 11 220-239 74 74 74 133 133 133 F F F 12 240-259 74 74 74 134 134 134 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 77 77 77 140 140 140 F F F 15 300-319 76 76 76 137 137 137 F F F 16 320-339 77 77 77 140 140 140 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 76 76 76 138 138 138 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 76 76 76 138 138 138 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 77 77 77 140 140 140 F F F 25 500-519 76 76 76 137 137 137 F F F 26 520-539 69 69 69 128 128 128 F F F 27 540-559 73 73 73 132 132 132 F F F 28 560-579 76 76 76 137 137 137 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 77, Max flush: 140 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.889317912Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.94768591Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.948705827Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.949003075Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.949077427Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.94909834Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"Crucible d88922fe-9495-457e-aa63-158bdbc33b77 has session id: 5090a90d-180c-472e-ab4a-08307fda974b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.9497288Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Upstairs opts: Upstairs UUID: d88922fe-9495-457e-aa63-158bdbc33b77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.949786237Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Crucible stats registered with UUID: d88922fe-9495-457e-aa63-158bdbc33b77","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.949804695Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:13.949820487Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.95049633Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950536162Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"d88922fe-9495-457e-aa63-158bdbc33b77 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950665616Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950687672Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950719687Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950736031Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950753476Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950769369Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950786654Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.950802707Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.951043259Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.951108282Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40173)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.951129536Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48717)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.951149136Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44523)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.951641884Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.952225067Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.952266833Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.952373641Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.953310969Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.953344307Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.953364338Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954238887Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954268398Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[0]R flush_numbers[0..12]: [135, 136, 131, 140, 132, 130, 130, 112, 139, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954380016Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[0]R generation[0..12]: [75, 76, 72, 77, 73, 71, 71, 64, 76, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954403955Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954429517Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[1]R flush_numbers[0..12]: [135, 136, 131, 140, 132, 130, 130, 112, 139, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954448466Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[1]R generation[0..12]: [75, 76, 72, 77, 73, 71, 71, 64, 76, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954466703Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954485612Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[2]R flush_numbers[0..12]: [135, 136, 131, 140, 132, 130, 130, 112, 139, 135, 121, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.95450421Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[2]R generation[0..12]: [75, 76, 72, 77, 73, 71, 71, 64, 76, 75, 68, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954522327Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954541917Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954559022Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954576538Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954593322Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954609866Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"":"downstairs","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954627182Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954642613Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"0","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954660219Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"1","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954677685Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"client":"2","":"downstairs","session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954698177Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.95471433Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"d88922fe-9495-457e-aa63-158bdbc33b77 is now active with session: 5090a90d-180c-472e-ab4a-08307fda974b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954731405Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954747368Z","hostname":"bmat-EVT22200007-0000589c","pid":1499,"session_id":"5090a90d-180c-472e-ab4a-08307fda974b"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954763891Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} {"msg":"Activated sub_volume d88922fe-9495-457e-aa63-158bdbc33b77","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:13.954779303Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 12288 data: 5 5 5 02/30 Write block 110 len 16384 data: 7 7 8 11 03/30 Write block 550 len 4096 data: 7 04/30 Flush 05/30 Write block 137 len 32768 data: 8 9 9 9 8 8 11 12 06/30 Read block 24 len 28672 07/30 Write block 296 len 36864 data: 11 12 10 8 8 7 7 6 5 08/30 Read block 313 len 20480 09/30 Read block 467 len 24576 10/30 Read block 5 len 40960 11/30 Flush 12/30 Write block 195 len 32768 data: 8 7 7 7 5 7 6 6 13/30 Flush 14/30 Write block 240 len 40960 data: 5 7 9 9 10 11 10 10 8 9 15/30 Write block 75 len 24576 data: 5 3 3 8 8 8 16/30 Write block 552 len 16384 data: 7 8 8 6 17/30 Write block 498 len 40960 data: 5 6 7 7 7 6 7 6 8 9 18/30 Write block 500 len 28672 data: 8 8 8 7 8 7 9 19/30 Flush 20/30 Write block 205 len 24576 data: 6 7 6 5 5 6 21/30 Write block 194 len 4096 data: 7 22/30 Read block 242 len 40960 23/30 Read block 371 len 28672 24/30 Write block 2 len 32768 data: 4 5 5 6 6 5 4 5 25/30 Write block 208 len 4096 data: 6 26/30 Write block 370 len 40960 data: 5 8 9 13 17 15 14 11 11 11 27/30 Write block 562 len 4096 data: 6 28/30 Read block 194 len 8192 29/30 Write block 539 len 16384 data: 6 6 6 6 30/30 Write block 39 len 16384 data: 8 10 9 10 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:13.981442845Z","hostname":"bmat-EVT22200007-0000589c","pid":1499} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 135 135 135 T T T 1 020-039 78 78 78 141 141 141 T T T 2 040-059 72 72 72 131 131 131 T T T 3 060-079 78 78 78 144 144 144 F F F 4 080-099 78 78 78 144 144 144 F F F 5 100-119 78 78 78 141 141 141 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 143 143 143 T T T 10 200-219 78 78 78 143 143 143 T T T 11 220-239 74 74 74 133 133 133 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 78 78 78 142 142 142 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 77 77 77 140 140 140 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 76 76 76 138 138 138 T T T 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 76 76 76 138 138 138 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 78 78 78 144 144 144 F F F 25 500-519 78 78 78 144 144 144 F F F 26 520-539 69 69 69 128 128 128 T T T 27 540-559 78 78 78 144 144 144 T T T 28 560-579 76 76 76 137 137 137 T T T 29 580-599 76 76 76 138 138 138 F F F Max gen: 78, Max flush: 144 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 145 145 145 F F F 1 020-039 79 79 79 145 145 145 F F F 2 040-059 79 79 79 145 145 145 F F F 3 060-079 78 78 78 144 144 144 F F F 4 080-099 78 78 78 144 144 144 F F F 5 100-119 78 78 78 141 141 141 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 79 79 79 145 145 145 F F F 10 200-219 79 79 79 145 145 145 F F F 11 220-239 74 74 74 133 133 133 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 73 73 73 132 132 132 F F F 14 280-299 78 78 78 142 142 142 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 77 77 77 140 140 140 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 79 79 79 145 145 145 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 76 76 76 138 138 138 F F F 23 460-479 66 66 66 119 119 119 F F F 24 480-499 78 78 78 144 144 144 F F F 25 500-519 78 78 78 144 144 144 F F F 26 520-539 79 79 79 145 145 145 F F F 27 540-559 79 79 79 145 145 145 F F F 28 560-579 79 79 79 145 145 145 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 79, Max flush: 145 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.352656937Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.412086337Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.413800524Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414061508Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414141582Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414171894Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"Crucible cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f has session id: 15930743-db50-4e2d-bcc7-8339ba8054d7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414795219Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Upstairs opts: Upstairs UUID: cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414931007Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Crucible stats registered with UUID: cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.414960517Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:18.414977011Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.415681242Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.415722116Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.415883365Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.415920822Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.415974833Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.41604202Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416074396Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.41610054Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416165833Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416197418Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416225576Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.41624743Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42938)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416456288Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42163)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416735038Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46919)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.416808859Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.417750375Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.418371747Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.418453514Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.419136561Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.41921888Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.419245364Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.41996659Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420022694Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 145, 144, 144, 141, 142, 142, 139, 145, 145, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420395697Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 78, 78, 78, 78, 78, 76, 79, 79, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420426971Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420449317Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 145, 144, 144, 141, 142, 142, 139, 145, 145, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420470741Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 78, 78, 78, 78, 78, 76, 79, 79, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420491834Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420513378Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 145, 144, 144, 141, 142, 142, 139, 145, 145, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420534982Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 78, 78, 78, 78, 78, 76, 79, 79, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420555575Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420576988Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420601569Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.42062169Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420641911Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420661251Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"":"downstairs","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420681292Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420699178Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"0","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.4207194Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"1","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420739381Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"client":"2","":"downstairs","session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420759722Z",Wait for a query_work_queue command to finish before sending IO "hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420794754Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f is now active with session: 15930743-db50-4e2d-bcc7-8339ba8054d7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420814574Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420832551Z","hostname":"bmat-EVT22200007-0000589c","pid":1507,"session_id":"15930743-db50-4e2d-bcc7-8339ba8054d7"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.42085131Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} {"msg":"Activated sub_volume cdd5ddb5-1dd4-4aeb-91af-062794ac9b1f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:18.420868094Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 3 len 24576 data: 6 6 7 7 6 5 02/30 Flush 03/30 Write block 223 len 36864 data: 8 8 8 5 3 5 4 4 5 04/30 Read block 58 len 32768 05/30 Read block 238 len 40960 06/30 Read block 480 len 40960 07/30 Write block 489 len 20480 data: 6 5 4 3 5 08/30 Flush 09/30 Write block 328 len 4096 data: 6 10/30 Flush 11/30 Write block 110 len 36864 data: 8 8 9 12 9 10 11 9 8 12/30 Flush 13/30 Write block 44 len 8192 data: 9 8 14/30 Flush 15/30 Write block 152 len 16384 data: 4 3 4 5 16/30 Write block 13 len 24576 data: 6 7 4 3 3 3 17/30 Write block 200 len 8192 data: 8 7 18/30 Write block 467 len 40960 data: 7 8 9 8 6 4 3 2 3 5 19/30 Read block 331 len 4096 20/30 Read block 144 len 16384 21/30 Write block 446 len 4096 data: 8 22/30 Read block 547 len 32768 23/30 Write block 330 len 16384 data: 7 7 7 6 24/30 Read block 568 len 28672 25/30 Write block 269 len 28672 data: 9 9 8 6 6 6 6 26/30 Flush 27/30 Write block 34 len 32768 data: 9 9 7 7 8 9 11 10 28/30 Write block 544 len 4096 data: 8 29/30 Read block 78 len 12288 30/30 Read block 138 len 28672 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:18.454354245Z","hostname":"bmat-EVT22200007-0000589c","pid":1507} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 79 79 79 145 145 145 T T T 2 040-059 80 80 80 150 150 150 T T T 3 060-079 78 78 78 144 144 144 F F F 4 080-099 78 78 78 144 144 144 F F F 5 100-119 80 80 80 149 149 149 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 80 80 80 151 151 151 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 79 79 79 145 145 145 F F F 10 200-219 80 80 80 151 151 151 F F F 11 220-239 80 80 80 147 147 147 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 78 78 78 142 142 142 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 79 79 79 145 145 145 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 80 80 80 151 151 151 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 80 80 80 147 147 147 F F F 25 500-519 78 78 78 144 144 144 F F F 26 520-539 79 79 79 145 145 145 F F F 27 540-559 79 79 79 145 145 145 T T T 28 560-579 79 79 79 145 145 145 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 80, Max flush: 151 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 81 81 81 152 152 152 F F F 2 040-059 81 81 81 152 152 152 F F F 3 060-079 78 78 78 144 144 144 F F F 4 080-099 78 78 78 144 144 144 F F F 5 100-119 80 80 80 149 149 149 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 80 80 80 151 151 151 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 79 79 79 145 145 145 F F F 10 200-219 80 80 80 151 151 151 F F F 11 220-239 80 80 80 147 147 147 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 78 78 78 142 142 142 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 79 79 79 145 145 145 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 80 80 80 151 151 151 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 80 80 80 147 147 147 F F F 25 500-519 78 78 78 144 144 144 F F F 26 520-539 79 79 79 145 145 145 F F F 27 540-559 81 81 81 152 152 152 F F F 28 560-579 79 79 79 145 145 145 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 81, Max flush: 152 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.7717565Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.825472821Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.826250281Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.82634741Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.826369886Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.826392342Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"Crucible 880118d9-f5fa-4f11-a13f-ca8ae28f5cf5 has session id: 3438e5ee-b93c-4219-8ff0-70f711759328","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.827274457Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Upstairs opts: Upstairs UUID: 880118d9-f5fa-4f11-a13f-ca8ae28f5cf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.827328878Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Crucible stats registered with UUID: 880118d9-f5fa-4f11-a13f-ca8ae28f5cf5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.827350522Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:22.827376716Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.827982094Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828014731Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"880118d9-f5fa-4f11-a13f-ca8ae28f5cf5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828159417Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828183827Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828218739Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828239301Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828261356Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828281367Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828303061Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828323122Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828349436Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828377704Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:22.828553474Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:22.82858614Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:22.828617374Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"ds_connection connected from Ok(127.0.0.1:64069)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828667056Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.828690995Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"ds_connection connected from Ok(127.0.0.1:54321)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.82885948Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.829896742Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.830095809Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.830748023Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.830887719Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.831563281Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:22.8315967Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.830677834Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"ds_connection connected from Ok(127.0.0.1:34714)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.831040766Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.832227995Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.832898868Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[0]R flush_numbers[0..12]: [151, 152, 152, 144, 144, 149, 142, 151, 139, 145, 151, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833753757Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[0]R generation[0..12]: [80, 81, 81, 78, 78, 80, 78, 80, 76, 79, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833897391Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833915919Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[1]R flush_numbers[0..12]: [151, 152, 152, 144, 144, 149, 142, 151, 139, 145, 151, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833933004Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[1]R generation[0..12]: [80, 81, 81, 78, 78, 80, 78, 80, 76, 79, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833949157Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833965961Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[2]R flush_numbers[0..12]: [151, 152, 152, 144, 144, 149, 142, 151, 139, 145, 151, 147]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.833982575Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[2]R generation[0..12]: [80, 81, 81, 78, 78, 80, 78, 80, 76, 79, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834013057Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834030724Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834046616Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834062408Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.83407769Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834093352Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"":"downstairs","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834108954Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834122912Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"0","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834143404Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"1","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834159647Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"client":"2","":"downstairs","session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834175279Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834191673Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"880118d9-f5fa-4f11-a13f-ca8ae28f5cf5 is now active with session: 3438e5ee-b93c-4219-8ff0-70f711759328","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834207445Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834221614Z","hostname":"bmat-EVT22200007-0000589c","pid":1515,"session_id":"3438e5ee-b93c-4219-8ff0-70f711759328"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834236324Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} {"msg":"Activated sub_volume 880118d9-f5fa-4f11-a13f-ca8ae28f5cf5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:32.834462387Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 554 len 36864 data: 9 7 9 9 8 8 6 8 7 02/30 Read block 7 len 36864 03/30 Read block 12 len 36864 04/30 Read block 28 len 4096 05/30 Flush 06/30 Write block 98 len 12288 data: 7 8 7 07/30 Write block 283 len 16384 data: 5 3 3 4 08/30 Read block 524 len 32768 09/30 Write block 562 len 36864 data: 8 4 5 6 6 8 9 7 9 10/30 Write block 487 len 28672 data: 9 9 7 6 5 4 6 11/30 Read block 169 len 8192 12/30 Read block 274 len 28672 13/30 Read block 490 len 36864 14/30 Flush 15/30 Write block 50 len 24576 data: 5 5 8 9 8 7 16/30 Write block 212 len 16384 data: 4 3 3 3 17/30 Read block 475 len 16384 18/30 Read block 492 len 28672 19/30 Flush 20/30 Write block 52 len 36864 data: 9 10 9 8 7 9 8 7 9 21/30 Flush 22/30 Write block 519 len 16384 data: 4 4 5 5 23/30 Flush 24/30 Flush 25/30 Write block 577 len 12288 data: 5 5 5 26/30 Read block 302 len 16384 27/30 Write block 93 len 20480 data: 9 8 7 8 8 28/30 Write block 170 len 8192 data: 8 9 29/30 Write block 217 len 36864 data: 8 6 7 7 7 9 9 9 9 30/30 Write block 522 len 40960 data: 6 5 6 6 5 5 6 7 7 7 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 4 2 8 Done 4 2 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:32.864488429Z","hostname":"bmat-EVT22200007-0000589c","pid":1515} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 81 81 81 152 152 152 F F F 2 040-059 82 82 82 156 156 156 F F F 3 060-079 82 82 82 156 156 156 F F F 4 080-099 82 82 82 154 154 154 T T T 5 100-119 82 82 82 154 154 154 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 80 80 80 151 151 151 F F F 8 160-179 76 76 76 139 139 139 T T T 9 180-199 79 79 79 145 145 145 F F F 10 200-219 82 82 82 155 155 155 T T T 11 220-239 80 80 80 147 147 147 T T T 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 82 82 82 154 154 154 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 79 79 79 145 145 145 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 80 80 80 151 151 151 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 82 82 82 157 157 157 F F F 26 520-539 82 82 82 157 157 157 T T F <--- 27 540-559 82 82 82 153 153 153 F F F 28 560-579 82 82 82 154 154 154 T T T 29 580-599 76 76 76 138 138 138 F F F Max gen: 82, Max flush: 157 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.97/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.97/src/lib.rs:692:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 81 81 81 152 152 152 F F F 2 040-059 82 82 82 156 156 156 F F F 3 060-079 82 82 82 156 156 156 F F F 4 080-099 83 83 83 158 158 158 F F F 5 100-119 82 82 82 154 154 154 F F F 6 120-139 78 78 78 142 142 142 F F F 7 140-159 80 80 80 151 151 151 F F F 8 160-179 83 83 83 158 158 158 F F F 9 180-199 79 79 79 145 145 145 F F F 10 200-219 83 83 83 158 158 158 F F F 11 220-239 83 83 83 158 158 158 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 82 82 82 154 154 154 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 77 77 77 140 140 140 F F F 18 360-379 79 79 79 145 145 145 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 F F F 22 440-459 80 80 80 151 151 151 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 82 82 82 157 157 157 F F F 26 520-539 83 83 83 158 158 158 F F F 27 540-559 82 82 82 153 153 153 F F F 28 560-579 83 83 83 158 158 158 F F F 29 580-599 76 76 76 138 138 138 F F F Max gen: 83, Max flush: 158 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.34691682Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:37.400582437Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.401011004Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.439821719Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.441705112Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.441812582Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.441843645Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.441870861Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"Crucible 71270b47-d6c2-46b0-90d9-fb82d71b2b7a has session id: a29e0eba-038f-4a52-8595-ac539f1c6898","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.442982425Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Upstairs opts: Upstairs UUID: 71270b47-d6c2-46b0-90d9-fb82d71b2b7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.443045144Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Crucible stats registered with UUID: 71270b47-d6c2-46b0-90d9-fb82d71b2b7a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.443068622Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:37.443087751Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.443890483Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.443924352Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"71270b47-d6c2-46b0-90d9-fb82d71b2b7a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444055511Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444079871Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444116636Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444139022Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444163121Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444185086Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444209235Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444230759Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444253776Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444331285Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"ds_connection connected from Ok(127.0.0.1:36879)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.44458313Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"ds_connection connected from Ok(127.0.0.1:33269)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.444830296Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"ds_connection connected from Ok(127.0.0.1:64342)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.445011667Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.445588778Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.445669272Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.445976311Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.446346518Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.446657745Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.446951105Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447451148Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447633581Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[0]R flush_numbers[0..12]: [151, 152, 156, 156, 158, 154, 142, 151, 158, 145, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447876158Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[0]R generation[0..12]: [80, 81, 82, 82, 83, 82, 78, 80, 83, 79, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447902261Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447921911Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[1]R flush_numbers[0..12]: [151, 152, 156, 156, 158, 154, 142, 151, 158, 145, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447940549Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[1]R generation[0..12]: [80, 81, 82, 82, 83, 82, 78, 80, 83, 79, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447959017Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447977545Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[2]R flush_numbers[0..12]: [151, 152, 156, 156, 158, 154, 142, 151, 158, 145, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.447996193Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[2]R generation[0..12]: [80, 81, 82, 82, 83, 82, 78, 80, 83, 79, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448020112Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448039211Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448057248Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448075105Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.44809257Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448109866Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"":"downstairs","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448129716Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448148074Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"0","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448168065Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"1","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448188056Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"client":"2","":"downstairs","session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448208437Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2025-03-23T06:06:37.448227046Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"71270b47-d6c2-46b0-90d9-fb82d71b2b7a is now active with session: a29e0eba-038f-4a52-8595-ac539f1c6898","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448262197Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448280655Z","hostname":"bmat-EVT22200007-0000589c","pid":1523,"session_id":"a29e0eba-038f-4a52-8595-ac539f1c6898"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448299384Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} {"msg":"Activated sub_volume 71270b47-d6c2-46b0-90d9-fb82d71b2b7a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:37.448316448Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 12288 data: 9 7 6 02/30 Flush 03/30 Write block 503 len 32768 data: 8 9 8 10 10 10 9 9 04/30 Flush 05/30 Write block 343 len 40960 data: 10 11 9 7 7 6 4 4 3 5 06/30 Read block 536 len 40960 07/30 Write block 510 len 32768 data: 10 9 8 10 9 8 8 8 08/30 Write block 139 len 12288 data: 10 10 9 09/30 Read block 0 len 36864 10/30 Read block 43 len 40960 11/30 Write block 435 len 8192 data: 8 8 12/30 Read block 128 len 36864 13/30 Write block 435 len 40960 data: 9 9 7 8 7 7 7 7 6 8 14/30 Read block 84 len 32768 15/30 Read block 498 len 4096 16/30 Read block 472 len 4096 17/30 Read block 519 len 4096 18/30 Write block 78 len 16384 data: 9 9 9 6 19/30 Write block 515 len 32768 data: 9 9 9 8 5 5 6 7 20/30 Write block 186 len 16384 data: 7 7 6 5 21/30 Read block 84 len 24576 22/30 Write block 291 len 4096 data: 5 23/30 Read block 250 len 28672 24/30 Read block 223 len 24576 25/30 Read block 214 len 4096 26/30 Read block 395 len 40960 27/30 Read block 507 len 32768 28/30 Read block 245 len 8192 29/30 Write block 526 len 12288 data: 6 6 7 30/30 Write block 577 len 20480 data: 6 6 6 5 5 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 24 25 25 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:37.483495112Z","hostname":"bmat-EVT22200007-0000589c","pid":1523} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 81 81 81 152 152 152 F F F 2 040-059 82 82 82 156 156 156 F F F 3 060-079 82 82 82 156 156 156 T T T 4 080-099 83 83 83 158 158 158 T T T 5 100-119 82 82 82 154 154 154 F F F 6 120-139 78 78 78 142 142 142 T T T 7 140-159 80 80 80 151 151 151 T T T 8 160-179 83 83 83 158 158 158 F F F 9 180-199 79 79 79 145 145 145 T T T 10 200-219 83 83 83 158 158 158 F F F 11 220-239 83 83 83 158 158 158 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 82 82 82 154 154 154 T T T 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 84 84 84 159 159 159 T T T 18 360-379 84 84 84 159 159 159 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 71 71 71 130 130 130 T T T 22 440-459 80 80 80 151 151 151 T T T 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 84 84 84 160 160 160 T T T 26 520-539 83 83 83 158 158 158 T T T 27 540-559 82 82 82 153 153 153 F F F 28 560-579 83 83 83 158 158 158 T T T 29 580-599 76 76 76 138 138 138 T T T Max gen: 84, Max flush: 160 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 151 151 151 F F F 1 020-039 81 81 81 152 152 152 F F F 2 040-059 82 82 82 156 156 156 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 85 85 85 161 161 161 F F F 5 100-119 82 82 82 154 154 154 F F F 6 120-139 85 85 85 161 161 161 F F F 7 140-159 85 85 85 161 161 161 F F F 8 160-179 83 83 83 158 158 158 F F F 9 180-199 85 85 85 161 161 161 F F F 10 200-219 83 83 83 158 158 158 F F F 11 220-239 83 83 83 158 158 158 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 F F F 14 280-299 85 85 85 161 161 161 F F F 15 300-319 78 78 78 142 142 142 F F F 16 320-339 80 80 80 151 151 151 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 84 84 84 159 159 159 F F F 19 380-399 75 75 75 135 135 135 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 85 85 85 161 161 161 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 85 85 85 161 161 161 F F F 27 540-559 82 82 82 153 153 153 F F F 28 560-579 85 85 85 161 161 161 F F F 29 580-599 85 85 85 161 161 161 F F F Max gen: 85, Max flush: 161 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.931027928Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.986473086Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.987400564Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.987487462Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.987508955Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.987528726Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"Crucible 0e041cd7-9682-4b74-9e98-088a68137653 has session id: 19fd1654-bbd8-4b3b-b4ed-c4998ab22087","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.98872327Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Upstairs opts: Upstairs UUID: 0e041cd7-9682-4b74-9e98-088a68137653, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.988825539Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Crucible stats registered with UUID: 0e041cd7-9682-4b74-9e98-088a68137653","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.988849979Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:41.988869198Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989566796Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989630115Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"0e041cd7-9682-4b74-9e98-088a68137653 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989710319Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989733878Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989766414Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989784291Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989802969Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989828411Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989847911Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989865176Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.989998599Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.99002358Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"ds_connection connected from Ok(127.0.0.1:41517)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.990151241Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"ds_connection connected from Ok(127.0.0.1:53508)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.990210022Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"ds_connection connected from Ok(127.0.0.1:59514)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.990672097Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.991162831Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.991386068Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.991583762Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.992037851Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.99207609Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.992662359Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.992791433Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.992889163Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[0]R flush_numbers[0..12]: [151, 152, 156, 161, 161, 154, 161, 161, 158, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993372111Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[0]R generation[0..12]: [80, 81, 82, 85, 85, 82, 85, 85, 83, 85, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993399688Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993419238Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[1]R flush_numbers[0..12]: [151, 152, 156, 161, 161, 154, 161, 161, 158, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993437876Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[1]R generation[0..12]: [80, 81, 82, 85, 85, 82, 85, 85, 83, 85, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993455953Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993474741Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[2]R flush_numbers[0..12]: [151, 152, 156, 161, 161, 154, 161, 161, 158, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993493329Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[2]R generation[0..12]: [80, 81, 82, 85, 85, 82, 85, 85, 83, 85, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993511396Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993530064Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993546819Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993563212Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993579235Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993600789Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"":"downstairs","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993617593Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993632815Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"0","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.99365005Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"1","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993666854Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"client":"2","":"downstairs","session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.99368435Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993699812Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"0e041cd7-9682-4b74-9e98-088a68137653 is now active with session: 19fd1654-bbd8-4b3b-b4ed-c4998ab22087","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993716165Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993731236Z","hostname":"bmat-EVT22200007-0000589c","pid":1531,"session_id":"19fd1654-bbd8-4b3b-b4ed-c4998ab22087"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993746998Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} {"msg":"Activated sub_volume 0e041cd7-9682-4b74-9e98-088a68137653","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:41.993761277Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 139 len 4096 data: 11 02/30 Write block 34 len 20480 data: 10 10 8 8 9 03/30 Read block 71 len 8192 04/30 Write block 597 len 12288 data: 6 3 2 05/30 Write block 128 len 8192 data: 4 5 06/30 Write block 106 len 8192 data: 7 6 07/30 Read block 231 len 40960 08/30 Write block 37 len 28672 data: 9 10 10 12 11 11 11 09/30 Flush 10/30 Write block 445 len 20480 data: 7 9 7 7 7 11/30 Read block 200 len 12288 12/30 Write block 318 len 20480 data: 8 7 8 6 6 13/30 Read block 498 len 36864 14/30 Write block 65 len 12288 data: 6 6 7 15/30 Write block 575 len 40960 data: 6 7 7 7 7 6 6 5 7 7 16/30 Write block 390 len 28672 data: 10 14 12 11 11 11 12 17/30 Write block 202 len 20480 data: 7 7 7 7 8 18/30 Write block 536 len 4096 data: 5 19/30 Write block 5 len 16384 data: 8 8 7 6 20/30 Write block 95 len 4096 data: 8 21/30 Read block 184 len 28672 22/30 Read block 128 len 12288 23/30 Write block 83 len 12288 data: 5 6 7 24/30 Flush 25/30 Write block 277 len 16384 data: 6 7 8 9 26/30 Write block 132 len 40960 data: 5 5 6 7 8 9 10 12 11 10 27/30 Write block 201 len 20480 data: 8 8 8 8 8 28/30 Read block 231 len 32768 29/30 Write block 311 len 16384 data: 3 3 3 5 30/30 Write block 21 len 24576 data: 2 6 6 6 6 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:42.02125072Z","hostname":"bmat-EVT22200007-0000589c","pid":1531} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 163 163 163 F F F 1 020-039 86 86 86 162 162 162 T T T 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 86 86 86 163 163 163 F F F 5 100-119 86 86 86 162 162 162 F F F 6 120-139 86 86 86 162 162 162 T T T 7 140-159 85 85 85 161 161 161 T T T 8 160-179 83 83 83 158 158 158 F F F 9 180-199 85 85 85 161 161 161 F F F 10 200-219 86 86 86 163 163 163 T T T 11 220-239 83 83 83 158 158 158 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 80 80 80 151 151 151 T T T 14 280-299 85 85 85 161 161 161 T T T 15 300-319 86 86 86 163 163 163 T T T 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 84 84 84 159 159 159 F F F 19 380-399 86 86 86 163 163 163 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 86 86 86 163 163 163 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 86 86 86 163 163 163 F F F 27 540-559 82 82 82 153 153 153 F F F 28 560-579 86 86 86 163 163 163 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 86, Max flush: 163 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 163 163 163 F F F 1 020-039 87 87 87 164 164 164 F F F 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 86 86 86 163 163 163 F F F 5 100-119 86 86 86 162 162 162 F F F 6 120-139 87 87 87 164 164 164 F F F 7 140-159 87 87 87 164 164 164 F F F 8 160-179 83 83 83 158 158 158 F F F 9 180-199 85 85 85 161 161 161 F F F 10 200-219 87 87 87 164 164 164 F F F 11 220-239 83 83 83 158 158 158 F F F 12 240-259 78 78 78 144 144 144 F F F 13 260-279 87 87 87 164 164 164 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 87 87 87 164 164 164 F F F 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 84 84 84 159 159 159 F F F 19 380-399 86 86 86 163 163 163 F F F 20 400-419 66 66 66 115 115 115 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 86 86 86 163 163 163 F F F 23 460-479 80 80 80 151 151 151 F F F 24 480-499 82 82 82 154 154 154 F F F 25 500-519 85 85 85 161 161 161 F F F 26 520-539 86 86 86 163 163 163 F F F 27 540-559 82 82 82 153 153 153 F F F 28 560-579 86 86 86 163 163 163 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 87, Max flush: 164 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.372853227Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.426187366Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.426947982Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.427009869Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.427060012Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.427081646Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"Crucible eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a has session id: 1cb71de0-a843-4bd7-a732-fcabb6b4e7f8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.428096162Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Upstairs opts: Upstairs UUID: eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.428216558Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Crucible stats registered with UUID: eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.428241469Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:46.428258664Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429129315Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429184158Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429200261Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429216083Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429245754Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429262819Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429281326Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.4292976Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429315406Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429331238Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429531759Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429614939Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64168)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.429645852Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48525)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.42974737Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42126)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.430179063Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.430618031Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.430841869Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.431190793Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.431426876Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.43180538Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.432136758Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.432174566Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.432589284Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[0]R flush_numbers[0..12]: [163, 164, 162, 163, 163, 162, 164, 164, 158, 161, 164, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.432953179Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[0]R generation[0..12]: [86, 87, 86, 86, 86, 86, 87, 87, 83, 85, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433001818Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433033663Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[1]R flush_numbers[0..12]: [163, 164, 162, 163, 163, 162, 164, 164, 158, 161, 164, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433062913Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[1]R generation[0..12]: [86, 87, 86, 86, 86, 86, 87, 87, 83, 85, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433091301Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433119629Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[2]R flush_numbers[0..12]: [163, 164, 162, 163, 163, 162, 164, 164, 158, 161, 164, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433148619Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[2]R generation[0..12]: [86, 87, 86, 86, 86, 86, 87, 87, 83, 85, 87, 83]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:06:46.433188821Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433225877Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433253122Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433279877Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433307975Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433334439Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"":"downstairs","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433361614Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433386385Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"0","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433414583Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"1","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433441538Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"client":"2","":"downstairs","session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433468423Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433493113Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a is now active with session: 1cb71de0-a843-4bd7-a732-fcabb6b4e7f8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433519568Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433552094Z","hostname":"bmat-EVT22200007-0000589c","pid":1539,"session_id":"1cb71de0-a843-4bd7-a732-fcabb6b4e7f8"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433577807Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} {"msg":"Activated sub_volume eda4aaa3-c3db-44d7-bfc0-ed28dd59ae1a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:46.433601626Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 114 len 36864 data: 10 11 12 10 9 7 7 5 4 02/30 Write block 478 len 8192 data: 6 6 03/30 Read block 405 len 36864 04/30 Flush 05/30 Write block 527 len 40960 data: 7 8 8 8 8 6 4 4 4 6 06/30 Read block 178 len 40960 07/30 Flush 08/30 Write block 398 len 12288 data: 8 8 9 09/30 Write block 135 len 4096 data: 8 10/30 Flush 11/30 Write block 307 len 4096 data: 5 12/30 Read block 379 len 28672 13/30 Read block 441 len 4096 14/30 Read block 349 len 28672 15/30 Read block 518 len 32768 16/30 Read block 115 len 28672 17/30 Flush 18/30 Flush 19/30 Write block 447 len 40960 data: 8 8 8 6 5 5 5 4 5 8 20/30 Read block 433 len 8192 21/30 Write block 379 len 16384 data: 12 9 9 6 22/30 Write block 0 len 40960 data: 2 2 5 7 7 9 9 8 7 6 23/30 Write block 230 len 16384 data: 5 6 9 10 24/30 Read block 171 len 4096 25/30 Write block 546 len 12288 data: 6 8 8 26/30 Read block 22 len 16384 27/30 Write block 168 len 8192 data: 11 9 28/30 Write block 497 len 28672 data: 4 6 7 9 9 9 9 29/30 Write block 561 len 36864 data: 9 9 5 6 7 7 9 10 8 30/30 Write block 257 len 16384 data: 11 9 8 5 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:46.461309615Z","hostname":"bmat-EVT22200007-0000589c","pid":1539} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 163 163 163 T T T 1 020-039 87 87 87 164 164 164 F F F 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 86 86 86 163 163 163 F F F 5 100-119 88 88 88 165 165 165 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 87 87 87 164 164 164 F F F 8 160-179 83 83 83 158 158 158 T T T 9 180-199 85 85 85 161 161 161 F F F 10 200-219 87 87 87 164 164 164 F F F 11 220-239 83 83 83 158 158 158 T T T 12 240-259 78 78 78 144 144 144 T T T 13 260-279 87 87 87 164 164 164 T T T 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 84 84 84 159 159 159 T T T 19 380-399 88 88 88 167 167 167 T T T 20 400-419 88 88 88 167 167 167 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 86 86 86 163 163 163 T T T 23 460-479 88 88 88 165 165 165 F F F 24 480-499 82 82 82 154 154 154 T T T 25 500-519 85 85 85 161 161 161 T T T 26 520-539 88 88 88 166 166 166 F F F 27 540-559 82 82 82 153 153 153 T T T 28 560-579 86 86 86 163 163 163 T T T 29 580-599 86 86 86 163 163 163 F F F Max gen: 88, Max flush: 168 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 169 169 169 F F F 1 020-039 87 87 87 164 164 164 F F F 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 86 86 86 163 163 163 F F F 5 100-119 88 88 88 165 165 165 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 87 87 87 164 164 164 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 85 85 85 161 161 161 F F F 10 200-219 87 87 87 164 164 164 F F F 11 220-239 89 89 89 169 169 169 F F F 12 240-259 89 89 89 169 169 169 F F F 13 260-279 89 89 89 169 169 169 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 89 89 89 169 169 169 F F F 19 380-399 89 89 89 169 169 169 F F F 20 400-419 88 88 88 167 167 167 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 89 89 89 169 169 169 F F F 23 460-479 88 88 88 165 165 165 F F F 24 480-499 89 89 89 169 169 169 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 89 89 89 169 169 169 F F F 28 560-579 89 89 89 169 169 169 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 89, Max flush: 169 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.846473176Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.905626642Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.906699006Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.906752435Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.906770983Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.906797277Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"Crucible e8d5527e-18f0-4f9a-ae01-0769afbf193c has session id: cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.908073488Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Upstairs opts: Upstairs UUID: e8d5527e-18f0-4f9a-ae01-0769afbf193c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.908116846Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Crucible stats registered with UUID: e8d5527e-18f0-4f9a-ae01-0769afbf193c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.90814309Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:50.908159544Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909015605Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909274515Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"e8d5527e-18f0-4f9a-ae01-0769afbf193c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909296239Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909312663Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909352164Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909368918Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909386194Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.909401605Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.90941888Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910011763Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910086967Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.91010788Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"ds_connection connected from Ok(127.0.0.1:53635)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910126378Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"ds_connection connected from Ok(127.0.0.1:40879)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910144275Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"ds_connection connected from Ok(127.0.0.1:56752)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.91016176Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910623635Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.910671213Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.911143599Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.911693234Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.911889305Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.912223468Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.912437206Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.912633297Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[0]R flush_numbers[0..12]: [169, 164, 162, 163, 163, 165, 167, 164, 169, 161, 164, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913044318Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[0]R generation[0..12]: [89, 87, 86, 86, 86, 88, 88, 87, 89, 85, 87, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913086955Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913106605Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[1]R flush_numbers[0..12]: [169, 164, 162, 163, 163, 165, 167, 164, 169, 161, 164, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913124492Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[1]R generation[0..12]: [89, 87, 86, 86, 86, 88, 88, 87, 89, 85, 87, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913141978Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913159664Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[2]R flush_numbers[0..12]: [169, 164, 162, 163, 163, 165, 167, 164, 169, 161, 164, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.91317722Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[2]R generation[0..12]: [89, 87, 86, 86, 86, 88, 88, 87, 89, 85, 87, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913194265Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913211811Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913227643Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913243415Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913258987Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913274499Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"":"downstairs","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913290221Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.9133045Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"0","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913321174Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"1","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913342949Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"client":"2","":"downstairs","session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913360104Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913375185Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"e8d5527e-18f0-4f9a-ae01-0769afbf193c is now active with session: cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913391037Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913405767Z","hostname":"bmat-EVT22200007-0000589c","pid":1547,"session_id":"cf3ea4c5-d862-4cc5-9bdf-7f3d03cf1f16"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913420718Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} {"msg":"Activated sub_volume e8d5527e-18f0-4f9a-ae01-0769afbf193c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:50.913434837Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 202 len 20480 data: 9 9 9 9 9 02/30 Write block 405 len 40960 data: 8 7 7 9 10 10 10 11 9 7 03/30 Flush 04/30 Write block 391 len 24576 data: 15 13 12 12 12 13 05/30 Read block 59 len 16384 06/30 Read block 95 len 8192 07/30 Read block 114 len 16384 08/30 Read block 375 len 32768 09/30 Read block 448 len 32768 10/30 Write block 92 len 28672 data: 8 10 9 9 9 9 8 11/30 Read block 14 len 36864 12/30 Read block 163 len 28672 13/30 Write block 180 len 4096 data: 6 14/30 Write block 108 len 20480 data: 8 7 9 9 10 15/30 Read block 329 len 32768 16/30 Read block 203 len 36864 17/30 Write block 6 len 40960 data: 10 9 8 7 5 4 6 7 8 5 18/30 Read block 21 len 16384 19/30 Read block 422 len 36864 20/30 Read block 169 len 16384 21/30 Read block 206 len 32768 22/30 Flush 23/30 Write block 372 len 4096 data: 10 24/30 Write block 80 len 28672 data: 10 7 7 6 7 8 6 25/30 Read block 486 len 4096 26/30 Read block 287 len 24576 27/30 Read block 530 len 24576 28/30 Write block 144 len 24576 data: 13 13 11 8 8 8 29/30 Write block 448 len 4096 data: 9 30/30 Write block 227 len 28672 data: 4 6 5 6 7 10 11 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 1 Done Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 3 7 Done 7 5 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:50.942528231Z","hostname":"bmat-EVT22200007-0000589c","pid":1547} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 87 87 87 164 164 164 F F F 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 90 90 90 171 171 171 T T T 5 100-119 90 90 90 171 171 171 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 87 87 87 164 164 164 T T T 8 160-179 89 89 89 169 169 169 F F F 9 180-199 90 90 90 171 171 171 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 89 89 89 169 169 169 T T T 12 240-259 89 89 89 169 169 169 F F F 13 260-279 89 89 89 169 169 169 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 89 89 89 169 169 169 T T T 19 380-399 90 90 90 171 171 171 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 89 89 89 169 169 169 T T T 23 460-479 88 88 88 165 165 165 F F F 24 480-499 89 89 89 169 169 169 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 89 89 89 169 169 169 F F F 28 560-579 89 89 89 169 169 169 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 90, Max flush: 171 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 87 87 87 164 164 164 F F F 2 040-059 86 86 86 162 162 162 F F F 3 060-079 86 86 86 163 163 163 F F F 4 080-099 91 91 91 172 172 172 F F F 5 100-119 90 90 90 171 171 171 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 90 90 90 171 171 171 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 91 91 91 172 172 172 F F F 12 240-259 89 89 89 169 169 169 F F F 13 260-279 89 89 89 169 169 169 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 86 86 86 163 163 163 F F F 17 340-359 85 85 85 161 161 161 F F F 18 360-379 91 91 91 172 172 172 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 88 88 88 165 165 165 F F F 24 480-499 89 89 89 169 169 169 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 89 89 89 169 169 169 F F F 28 560-579 89 89 89 169 169 169 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 91, Max flush: 172 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.280551486Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.335735059Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.336721638Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.336792012Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.336814438Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.336834449Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"Crucible 697f6603-666b-4e6c-9282-bd014b795394 has session id: fcb4b64e-75db-4d54-a125-dee0c58981b1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.337706343Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Upstairs opts: Upstairs UUID: 697f6603-666b-4e6c-9282-bd014b795394, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.337761726Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Crucible stats registered with UUID: 697f6603-666b-4e6c-9282-bd014b795394","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.33778354Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:55.337801768Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338631906Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338669593Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"697f6603-666b-4e6c-9282-bd014b795394 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338769558Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338792395Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338827476Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338853971Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338874603Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338893091Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338913182Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338931359Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338950769Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.338970028Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59577)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.339399928Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46530)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.339439649Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39641)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.339460391Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.340899375Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.341108142Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.341134066Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.341270745Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.341540206Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.341925465Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342176869Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342288748Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[0]R flush_numbers[0..12]: [171, 164, 162, 163, 172, 171, 167, 172, 169, 171, 170, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342764431Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[0]R generation[0..12]: [90, 87, 86, 86, 91, 90, 88, 91, 89, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342789913Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342808281Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[1]R flush_numbers[0..12]: [171, 164, 162, 163, 172, 171, 167, 172, 169, 171, 170, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342825857Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[1]R generation[0..12]: [90, 87, 86, 86, 91, 90, 88, 91, 89, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342842741Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342860247Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[2]R flush_numbers[0..12]: [171, 164, 162, 163, 172, 171, 167, 172, 169, 171, 170, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342877653Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[2]R generation[0..12]: [90, 87, 86, 86, 91, 90, 88, 91, 89, 90, 90, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342894337Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342911402Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342927354Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342950732Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342967216Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342983329Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"":"downstairs","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.342999472Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343014363Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"0","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343031257Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"1","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.34304715Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"client":"2","":"downstairs","session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343063423Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343078163Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"697f6603-666b-4e6c-9282-bd014b795394 is now active with session: fcb4b64e-75db-4d54-a125-dee0c58981b1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343094046Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343108675Z","hostname":"bmat-EVT22200007-0000589c","pid":1555,"session_id":"fcb4b64e-75db-4d54-a125-dee0c58981b1"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343123526Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} {"msg":"Activated sub_volume 697f6603-666b-4e6c-9282-bd014b795394","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:55.343137074Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 40960 data: 7 7 7 8 7 6 5 6 6 8 02/30 Read block 20 len 4096 03/30 Write block 430 len 36864 data: 7 8 10 11 9 10 10 8 9 04/30 Write block 74 len 32768 data: 4 6 4 4 10 10 11 8 05/30 Write block 88 len 20480 data: 6 7 9 8 9 06/30 Write block 41 len 40960 data: 12 12 12 10 9 9 8 6 6 6 07/30 Read block 17 len 16384 08/30 Read block 248 len 16384 09/30 Write block 366 len 28672 data: 11 7 7 5 6 9 11 10/30 Write block 102 len 4096 data: 7 11/30 Read block 1 len 28672 12/30 Write block 222 len 20480 data: 10 10 10 10 6 13/30 Write block 329 len 32768 data: 5 8 8 8 7 7 7 8 14/30 Write block 345 len 16384 data: 10 8 8 7 15/30 Read block 31 len 32768 16/30 Write block 469 len 12288 data: 10 9 7 17/30 Flush 18/30 Write block 549 len 12288 data: 9 8 7 19/30 Read block 166 len 20480 20/30 Write block 471 len 40960 data: 8 5 4 3 4 6 6 7 7 7 21/30 Read block 276 len 8192 22/30 Read block 255 len 28672 23/30 Write block 245 len 12288 data: 12 11 11 24/30 Read block 271 len 32768 25/30 Write block 34 len 8192 data: 11 11 26/30 Write block 359 len 28672 data: 10 8 7 5 7 9 11 27/30 Write block 181 len 4096 data: 5 28/30 Read block 252 len 8192 29/30 Write block 257 len 36864 data: 12 10 9 6 7 8 8 7 9 30/30 Write block 96 len 20480 data: 10 10 9 9 8 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:55.371579445Z","hostname":"bmat-EVT22200007-0000589c","pid":1555} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 92 92 92 173 173 173 T T T 2 040-059 92 92 92 173 173 173 F F F 3 060-079 92 92 92 173 173 173 F F F 4 080-099 92 92 92 173 173 173 T T T 5 100-119 92 92 92 173 173 173 T T T 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 90 90 90 171 171 171 T T T 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 89 89 89 169 169 169 T T T 13 260-279 89 89 89 169 169 169 T T T 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 92 92 92 173 173 173 F F F 17 340-359 92 92 92 173 173 173 T T T 18 360-379 92 92 92 173 173 173 T T T 19 380-399 90 90 90 171 171 171 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 92 92 92 173 173 173 T T T 24 480-499 89 89 89 169 169 169 T T T 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 89 89 89 169 169 169 T T T 28 560-579 89 89 89 169 169 169 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 92, Max flush: 173 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 92 92 92 173 173 173 F F F 3 060-079 92 92 92 173 173 173 F F F 4 080-099 93 93 93 174 174 174 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 93 93 93 174 174 174 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 93 93 93 174 174 174 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 F F F 16 320-339 92 92 92 173 173 173 F F F 17 340-359 93 93 93 174 174 174 F F F 18 360-379 93 93 93 174 174 174 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 93 93 93 174 174 174 F F F 24 480-499 93 93 93 174 174 174 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 89 89 89 169 169 169 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 93, Max flush: 174 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.760779896Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:59.814054523Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.814537992Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.851159909Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.851945847Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.851985207Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.852004186Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.852025119Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"Crucible 966908b8-07a0-48c5-bea2-100b3cda5c2e has session id: 3764b0c0-dde7-4bf2-a265-69d85fbb2344","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.853045366Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Upstairs opts: Upstairs UUID: 966908b8-07a0-48c5-bea2-100b3cda5c2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.853156965Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Crucible stats registered with UUID: 966908b8-07a0-48c5-bea2-100b3cda5c2e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.853177767Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:59.853256669Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854041595Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854076707Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"966908b8-07a0-48c5-bea2-100b3cda5c2e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854212114Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854232926Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854266495Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854284722Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854303972Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854321527Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854340296Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854357962Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854456223Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854480914Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"ds_connection connected from Ok(127.0.0.1:53255)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854629878Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"ds_connection connected from Ok(127.0.0.1:48275)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.854862304Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"ds_connection connected from Ok(127.0.0.1:48332)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.855088787Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.855482132Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.855748517Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.85596004Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.856726999Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.856799267Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.856954875Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857404044Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857588091Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[0]R flush_numbers[0..12]: [171, 174, 173, 173, 174, 174, 167, 172, 169, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857701002Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[0]R generation[0..12]: [90, 93, 92, 92, 93, 93, 88, 91, 89, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857730482Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857759111Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[1]R flush_numbers[0..12]: [171, 174, 173, 173, 174, 174, 167, 172, 169, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857780234Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[1]R generation[0..12]: [90, 93, 92, 92, 93, 93, 88, 91, 89, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857799062Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857817881Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[2]R flush_numbers[0..12]: [171, 174, 173, 173, 174, 174, 167, 172, 169, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857836158Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[2]R generation[0..12]: [90, 93, 92, 92, 93, 93, 88, 91, 89, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857854706Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857873585Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.8578909Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857908546Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857926032Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857943628Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"":"downstairs","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.857979481Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858023622Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"0","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858044284Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"1","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858064064Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"client":"2","":"downstairs","session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858087132Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858103595Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"966908b8-07a0-48c5-bea2-100b3cda5c2e is now active with session: 3764b0c0-dde7-4bf2-a265-69d85fbb2344","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858120961Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858136753Z","hostname":"bmat-EVT22200007-0000589c","pid":1563,"session_id":"3764b0c0-dde7-4bf2-a265-69d85fbb2344"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858153568Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} {"msg":"Activated sub_volume 966908b8-07a0-48c5-bea2-100b3cda5c2e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:06:59.858169139Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 410 len 28672 data: 11 11 12 10 8 7 5 02/30 Read block 201 len 8192 03/30 Read block 104 len 36864 04/30 Flush 05/30 Write block 401 len 20480 data: 9 7 8 9 9 06/30 Read block 232 len 32768 07/30 Read block 301 len 40960 08/30 Read block 66 len 16384 09/30 Write block 51 len 40960 data: 6 10 11 10 9 8 10 9 8 10 10/30 Read block 284 len 20480 11/30 Read block 492 len 20480 12/30 Write block 105 len 12288 data: 6 8 7 13/30 Read block 272 len 12288 14/30 Flush 15/30 Write block 194 len 16384 data: 8 9 8 8 16/30 Write block 319 len 28672 data: 8 9 7 7 5 6 6 17/30 Read block 31 len 12288 18/30 Write block 97 len 8192 data: 11 10 19/30 Read block 547 len 4096 20/30 Read block 119 len 20480 21/30 Read block 73 len 4096 22/30 Read block 212 len 16384 23/30 Read block 588 len 28672 24/30 Read block 395 len 32768 25/30 Read block 117 len 4096 26/30 Write block 59 len 40960 data: 9 11 9 6 6 6 7 7 8 7 27/30 Write block 330 len 20480 data: 9 9 9 8 8 28/30 Write block 565 len 24576 data: 8 8 10 11 9 10 29/30 Write block 105 len 24576 data: 7 9 8 9 8 10 30/30 Write block 307 len 16384 data: 6 4 3 4 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:06:59.887771344Z","hostname":"bmat-EVT22200007-0000589c","pid":1563} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 94 94 94 176 176 176 T T T 3 060-079 94 94 94 176 176 176 T T T 4 080-099 93 93 93 174 174 174 T T T 5 100-119 94 94 94 176 176 176 T T T 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 93 93 93 174 174 174 T T T 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 93 93 93 174 174 174 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 88 88 88 168 168 168 T T T 16 320-339 92 92 92 173 173 173 T T T 17 340-359 93 93 93 174 174 174 F F F 18 360-379 93 93 93 174 174 174 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 94 94 94 176 176 176 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 93 93 93 174 174 174 F F F 24 480-499 93 93 93 174 174 174 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 89 89 89 169 169 169 T T T 29 580-599 86 86 86 163 163 163 F F F Max gen: 94, Max flush: 176 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 95 95 95 177 177 177 F F F 3 060-079 95 95 95 177 177 177 F F F 4 080-099 95 95 95 177 177 177 F F F 5 100-119 95 95 95 177 177 177 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 89 89 89 169 169 169 F F F 9 180-199 95 95 95 177 177 177 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 93 93 93 174 174 174 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 95 95 95 177 177 177 F F F 16 320-339 95 95 95 177 177 177 F F F 17 340-359 93 93 93 174 174 174 F F F 18 360-379 93 93 93 174 174 174 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 94 94 94 176 176 176 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 93 93 93 174 174 174 F F F 24 480-499 93 93 93 174 174 174 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 95 95 95 177 177 177 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 95, Max flush: 177 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.290350978Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:04.344948623Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.345404916Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.383746621Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.384812602Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.384882656Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.384902606Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.38492385Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"Crucible a87c0f3b-cb44-4dd4-8128-6aa5d081f319 has session id: 301e0014-e551-4601-88b7-fd402373ef98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.386152734Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Upstairs opts: Upstairs UUID: a87c0f3b-cb44-4dd4-8128-6aa5d081f319, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.38622385Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Crucible stats registered with UUID: a87c0f3b-cb44-4dd4-8128-6aa5d081f319","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.386247087Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:04.386265014Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387200598Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387254288Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"a87c0f3b-cb44-4dd4-8128-6aa5d081f319 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387442593Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387468316Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387506514Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387526825Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.3875488Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387568992Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387590636Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387610817Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387695611Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387723358Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:34139)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.387956424Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:64711)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.388077763Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:43496)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.388639302Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.389011523Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.38904402Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.389560376Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.389939742Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.389983261Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.390520751Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.390815765Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.390929287Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[0]R flush_numbers[0..12]: [171, 174, 177, 177, 177, 177, 167, 172, 169, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391269322Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[0]R generation[0..12]: [90, 93, 95, 95, 95, 95, 88, 91, 89, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391296719Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391318744Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[1]R flush_numbers[0..12]: [171, 174, 177, 177, 177, 177, 167, 172, 169, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391343384Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[1]R generation[0..12]: [90, 93, 95, 95, 95, 95, 88, 91, 89, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.39136574Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391387394Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[2]R flush_numbers[0..12]: [171, 174, 177, 177, 177, 177, 167, 172, 169, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391408688Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[2]R generation[0..12]: [90, 93, 95, 95, 95, 95, 88, 91, 89, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391439821Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391462317Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391482398Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.39150282Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.39152237Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.39154195Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"":"downstairs","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391562172Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391580579Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"0","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391601081Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"1","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391621032Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"client":"2","":"downstairs","session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391641314Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391660212Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"a87c0f3b-cb44-4dd4-8128-6aa5d081f319 is now active with session: 301e0014-e551-4601-88b7-fd402373ef98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391679892Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391697839Z","hostname":"bmat-EVT22200007-0000589c","pid":1571,"session_id":"301e0014-e551-4601-88b7-fd402373ef98"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391716327Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} {"msg":"Activated sub_volume a87c0f3b-cb44-4dd4-8128-6aa5d081f319","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:04.391733352Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 4096 data: 9 02/30 Read block 388 len 24576 03/30 Read block 508 len 32768 04/30 Write block 80 len 8192 data: 12 9 05/30 Read block 152 len 40960 06/30 Read block 158 len 12288 07/30 Flush 08/30 Write block 163 len 36864 data: 12 11 9 8 11 12 10 9 10 09/30 Read block 337 len 32768 10/30 Flush 11/30 Write block 463 len 28672 data: 11 11 11 9 8 9 11 12/30 Read block 465 len 16384 13/30 Read block 300 len 16384 14/30 Write block 408 len 8192 data: 10 11 15/30 Read block 26 len 20480 16/30 Read block 571 len 20480 17/30 Read block 559 len 32768 18/30 Read block 419 len 40960 19/30 Write block 406 len 24576 data: 8 8 11 12 12 12 20/30 Write block 356 len 16384 data: 8 9 9 11 21/30 Read block 84 len 40960 22/30 Read block 195 len 8192 23/30 Read block 466 len 28672 24/30 Read block 238 len 36864 25/30 Read block 502 len 12288 26/30 Read block 125 len 8192 27/30 Read block 426 len 28672 28/30 Write block 45 len 12288 data: 10 10 9 29/30 Read block 382 len 4096 30/30 Read block 486 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:04.424725522Z","hostname":"bmat-EVT22200007-0000589c","pid":1571} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 95 95 95 177 177 177 T T T 3 060-079 95 95 95 177 177 177 F F F 4 080-099 96 96 96 178 178 178 F F F 5 100-119 95 95 95 177 177 177 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 96 96 96 179 179 179 F F F 9 180-199 95 95 95 177 177 177 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 93 93 93 174 174 174 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 96 96 96 178 178 178 F F F 16 320-339 95 95 95 177 177 177 F F F 17 340-359 93 93 93 174 174 174 T T T 18 360-379 93 93 93 174 174 174 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 94 94 94 176 176 176 T T T 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 93 93 93 174 174 174 T T T 24 480-499 93 93 93 174 174 174 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 95 95 95 177 177 177 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 96, Max flush: 179 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 97 97 97 180 180 180 F F F 3 060-079 95 95 95 177 177 177 F F F 4 080-099 96 96 96 178 178 178 F F F 5 100-119 95 95 95 177 177 177 F F F 6 120-139 88 88 88 167 167 167 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 96 96 96 179 179 179 F F F 9 180-199 95 95 95 177 177 177 F F F 10 200-219 90 90 90 170 170 170 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 93 93 93 174 174 174 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 96 96 96 178 178 178 F F F 16 320-339 95 95 95 177 177 177 F F F 17 340-359 97 97 97 180 180 180 F F F 18 360-379 93 93 93 174 174 174 F F F 19 380-399 90 90 90 171 171 171 F F F 20 400-419 97 97 97 180 180 180 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 97 97 97 180 180 180 F F F 24 480-499 93 93 93 174 174 174 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 95 95 95 177 177 177 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 97, Max flush: 180 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.762602908Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:08.817866866Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.81859752Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.858213102Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.860056824Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.860114542Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.860144423Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.860168883Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"Crucible c72a040e-92dd-4ea1-890b-6817579db1b7 has session id: b1874b44-df18-44cf-9443-053361d90417","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.861311411Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Upstairs opts: Upstairs UUID: c72a040e-92dd-4ea1-890b-6817579db1b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.861367355Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Crucible stats registered with UUID: c72a040e-92dd-4ea1-890b-6817579db1b7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.861390813Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:08.861411997Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862432755Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862467757Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"c72a040e-92dd-4ea1-890b-6817579db1b7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.86260633Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862633887Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862672125Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862694471Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862718309Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.862741096Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863289147Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863325231Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863456259Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863484377Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"ds_connection connected from Ok(127.0.0.1:36847)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863508165Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"ds_connection connected from Ok(127.0.0.1:37025)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.863531403Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"ds_connection connected from Ok(127.0.0.1:51905)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.86355459Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.864338344Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.864406714Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.864668479Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.865355315Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.865474348Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.865684939Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866468883Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.86652595Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[0]R flush_numbers[0..12]: [171, 174, 180, 177, 178, 177, 167, 172, 179, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866700888Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[0]R generation[0..12]: [90, 93, 97, 95, 96, 95, 88, 91, 96, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866744627Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866768556Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[1]R flush_numbers[0..12]: [171, 174, 180, 177, 178, 177, 167, 172, 179, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866790721Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[1]R generation[0..12]: [90, 93, 97, 95, 96, 95, 88, 91, 96, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866812095Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.86683385Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[2]R flush_numbers[0..12]: [171, 174, 180, 177, 178, 177, 167, 172, 179, 177, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866855514Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[2]R generation[0..12]: [90, 93, 97, 95, 96, 95, 88, 91, 96, 95, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866876447Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866897831Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866917381Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866949586Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866969507Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.866988877Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"":"downstairs","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867015832Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.8670344Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"0","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867055263Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"1","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867075765Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"client":"2","":"downstairs","session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867095946Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867114724Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"c72a040e-92dd-4ea1-890b-6817579db1b7 is now active with session: b1874b44-df18-44cf-9443-053361d90417","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867134455Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867152482Z","hostname":"bmat-EVT22200007-0000589c","pid":1579,"session_id":"b1874b44-df18-44cf-9443-053361d90417"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.86717127Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} {"msg":"Activated sub_volume c72a040e-92dd-4ea1-890b-6817579db1b7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:08.867188305Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 36864 data: 5 5 11 11 13 10 8 7 8 02/30 Read block 145 len 4096 03/30 Write block 311 len 24576 data: 4 4 4 6 6 7 04/30 Write block 177 len 36864 data: 7 9 8 7 6 3 3 4 6 05/30 Read block 218 len 28672 06/30 Read block 229 len 24576 07/30 Read block 502 len 16384 08/30 Flush 09/30 Write block 268 len 36864 data: 10 10 10 9 7 7 7 7 6 10/30 Write block 580 len 40960 data: 7 7 6 8 8 7 6 5 5 6 11/30 Read block 483 len 28672 12/30 Read block 121 len 12288 13/30 Read block 259 len 28672 14/30 Read block 406 len 16384 15/30 Write block 341 len 12288 data: 9 12 11 16/30 Flush 17/30 Write block 307 len 28672 data: 7 5 4 5 5 5 5 18/30 Write block 477 len 28672 data: 7 8 8 8 8 9 10 19/30 Flush 20/30 Write block 337 len 36864 data: 8 8 9 10 10 13 12 12 11 21/30 Read block 338 len 12288 22/30 Write block 12 len 20480 data: 7 8 9 6 4 23/30 Read block 127 len 12288 24/30 Write block 376 len 24576 data: 15 12 12 13 10 10 25/30 Write block 126 len 20480 data: 5 5 5 6 5 26/30 Read block 568 len 20480 27/30 Write block 331 len 16384 data: 10 10 9 9 28/30 Read block 32 len 20480 29/30 Read block 114 len 36864 30/30 Write block 208 len 28672 data: 7 6 7 4 5 4 4 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 9 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:08.901557693Z","hostname":"bmat-EVT22200007-0000589c","pid":1579} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 171 171 171 T T T 1 020-039 93 93 93 174 174 174 F F F 2 040-059 97 97 97 180 180 180 F F F 3 060-079 98 98 98 181 181 181 F F F 4 080-099 98 98 98 181 181 181 F F F 5 100-119 95 95 95 177 177 177 F F F 6 120-139 88 88 88 167 167 167 T T T 7 140-159 91 91 91 172 172 172 F F F 8 160-179 98 98 98 181 181 181 F F F 9 180-199 98 98 98 181 181 181 F F F 10 200-219 90 90 90 170 170 170 T T T 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 98 98 98 183 183 183 F F F 16 320-339 95 95 95 177 177 177 T T T 17 340-359 98 98 98 182 182 182 T T T 18 360-379 93 93 93 174 174 174 T T T 19 380-399 90 90 90 171 171 171 T T T 20 400-419 97 97 97 180 180 180 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 98 98 98 183 183 183 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 95 95 95 177 177 177 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 98, Max flush: 183 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 93 93 93 174 174 174 F F F 2 040-059 97 97 97 180 180 180 F F F 3 060-079 98 98 98 181 181 181 F F F 4 080-099 98 98 98 181 181 181 F F F 5 100-119 95 95 95 177 177 177 F F F 6 120-139 99 99 99 184 184 184 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 98 98 98 181 181 181 F F F 9 180-199 98 98 98 181 181 181 F F F 10 200-219 99 99 99 184 184 184 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 98 98 98 183 183 183 F F F 16 320-339 99 99 99 184 184 184 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 99 99 99 184 184 184 F F F 20 400-419 97 97 97 180 180 180 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 F F F 23 460-479 98 98 98 183 183 183 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 93 93 93 174 174 174 F F F 28 560-579 95 95 95 177 177 177 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 99, Max flush: 184 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.306741875Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:13.360760184Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.361203972Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.398214354Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.399943912Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.400001991Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.400022493Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40004541Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"Crucible 57e2961e-5864-40a0-b6ee-03584d362f6a has session id: a2c883d0-3712-42a8-8bce-103e35e62a01","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40129188Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Upstairs opts: Upstairs UUID: 57e2961e-5864-40a0-b6ee-03584d362f6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.401358988Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Crucible stats registered with UUID: 57e2961e-5864-40a0-b6ee-03584d362f6a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.401382526Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:13.401411234Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.402204107Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.402693889Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"57e2961e-5864-40a0-b6ee-03584d362f6a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.402982629Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403025237Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403067724Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403088877Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403111263Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403131404Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403153679Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403173941Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403199293Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403307995Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"ds_connection connected from Ok(127.0.0.1:50029)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403331804Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"ds_connection connected from Ok(127.0.0.1:49116)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403353819Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"ds_connection connected from Ok(127.0.0.1:36987)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.403375523Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.404107801Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.404168325Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.404193397Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.404876374Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.405137989Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.405274709Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40593029Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40602282Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[0]R flush_numbers[0..12]: [184, 174, 180, 181, 181, 177, 184, 172, 181, 181, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40604738Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[0]R generation[0..12]: [99, 93, 97, 98, 98, 95, 99, 91, 98, 98, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406073423Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406098404Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[1]R flush_numbers[0..12]: [184, 174, 180, 181, 181, 177, 184, 172, 181, 181, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406122123Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[1]R generation[0..12]: [99, 93, 97, 98, 98, 95, 99, 91, 98, 98, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40614519Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406168898Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[2]R flush_numbers[0..12]: [184, 174, 180, 181, 181, 177, 184, 172, 181, 181, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406203078Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[2]R generation[0..12]: [99, 93, 97, 98, 98, 95, 99, 91, 98, 98, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406226456Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406249954Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406325619Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.4063504Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406372254Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406394209Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"":"downstairs","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40646284Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:07:13.406485997Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"0","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.40652625Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"1","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406548826Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"client":"2","":"downstairs","session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406571713Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406651556Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"57e2961e-5864-40a0-b6ee-03584d362f6a is now active with session: a2c883d0-3712-42a8-8bce-103e35e62a01","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406673661Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406693492Z","hostname":"bmat-EVT22200007-0000589c","pid":1587,"session_id":"a2c883d0-3712-42a8-8bce-103e35e62a01"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406771802Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} {"msg":"Activated sub_volume 57e2961e-5864-40a0-b6ee-03584d362f6a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:13.406795601Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 8192 data: 6 12 02/30 Write block 574 len 20480 data: 7 7 8 8 8 03/30 Read block 279 len 40960 04/30 Read block 325 len 40960 05/30 Read block 129 len 8192 06/30 Write block 550 len 12288 data: 9 8 8 07/30 Read block 542 len 24576 08/30 Write block 527 len 8192 data: 8 9 09/30 Flush 10/30 Write block 101 len 20480 data: 6 8 6 7 8 11/30 Read block 307 len 20480 12/30 Write block 535 len 16384 data: 5 7 6 7 13/30 Flush 14/30 Write block 122 len 12288 data: 5 3 6 15/30 Read block 257 len 28672 16/30 Flush 17/30 Write block 167 len 20480 data: 12 13 11 10 11 18/30 Read block 94 len 36864 19/30 Read block 391 len 28672 20/30 Read block 450 len 36864 21/30 Write block 329 len 4096 data: 6 22/30 Write block 50 len 28672 data: 7 7 11 12 11 10 9 23/30 Read block 409 len 8192 24/30 Write block 171 len 40960 data: 12 11 12 11 10 8 8 10 9 8 25/30 Write block 103 len 12288 data: 7 8 9 26/30 Write block 378 len 20480 data: 13 14 11 11 7 27/30 Write block 442 len 8192 data: 8 7 28/30 Read block 74 len 20480 29/30 Write block 404 len 16384 data: 10 10 9 9 30/30 Read block 591 len 28672 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:13.437863664Z","hostname":"bmat-EVT22200007-0000589c","pid":1587} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 100 100 100 185 185 185 F F F 2 040-059 97 97 97 180 180 180 T T T 3 060-079 98 98 98 181 181 181 F F F 4 080-099 98 98 98 181 181 181 F F F 5 100-119 100 100 100 186 186 186 T T T 6 120-139 100 100 100 187 187 187 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 98 98 98 181 181 181 T T T 9 180-199 98 98 98 181 181 181 T T T 10 200-219 99 99 99 184 184 184 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 98 98 98 183 183 183 F F F 16 320-339 99 99 99 184 184 184 T T T 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 T T T 19 380-399 99 99 99 184 184 184 T T T 20 400-419 97 97 97 180 180 180 T T T 21 420-439 92 92 92 173 173 173 F F F 22 440-459 91 91 91 172 172 172 T T T 23 460-479 98 98 98 183 183 183 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 100 100 100 186 186 186 F F F 27 540-559 100 100 100 185 185 185 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 100, Max flush: 187 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 F F F 1 020-039 100 100 100 185 185 185 F F F 2 040-059 101 101 101 188 188 188 F F F 3 060-079 98 98 98 181 181 181 F F F 4 080-099 98 98 98 181 181 181 F F F 5 100-119 101 101 101 188 188 188 F F F 6 120-139 100 100 100 187 187 187 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 101 101 101 188 188 188 F F F 9 180-199 101 101 101 188 188 188 F F F 10 200-219 99 99 99 184 184 184 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 98 98 98 183 183 183 F F F 16 320-339 101 101 101 188 188 188 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 101 101 101 188 188 188 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 98 98 98 183 183 183 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 100 100 100 186 186 186 F F F 27 540-559 100 100 100 185 185 185 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 101, Max flush: 188 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.854325039Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:17.90669681Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.907128112Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.948187498Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.949492808Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.949560467Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.949583444Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.94960629Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"Crucible e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8 has session id: 48f2f16b-746e-42e3-8601-592e28afcd28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.950703966Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Upstairs opts: Upstairs UUID: e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.950787537Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Crucible stats registered with UUID: e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.950817949Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:17.950842079Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.951610751Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.951653068Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.951755147Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.951782683Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952652062Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.95269986Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952730582Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952751545Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.95277353Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952793321Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952814434Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952838022Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"ds_connection connected from Ok(127.0.0.1:61111)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952859246Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"ds_connection connected from Ok(127.0.0.1:46643)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952880239Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"ds_connection connected from Ok(127.0.0.1:37448)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.952901502Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.953587105Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.953669293Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.953696669Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.955003643Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.955042903Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.955268084Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.955709207Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.955838551Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[0]R flush_numbers[0..12]: [184, 185, 188, 181, 181, 188, 187, 172, 188, 188, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956389518Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[0]R generation[0..12]: [99, 100, 101, 98, 98, 101, 100, 91, 101, 101, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956418738Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956443028Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[1]R flush_numbers[0..12]: [184, 185, 188, 181, 181, 188, 187, 172, 188, 188, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956466496Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[1]R generation[0..12]: [99, 100, 101, 98, 98, 101, 100, 91, 101, 101, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956490305Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956514294Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[2]R flush_numbers[0..12]: [184, 185, 188, 181, 181, 188, 187, 172, 188, 188, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956537882Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[2]R generation[0..12]: [99, 100, 101, 98, 98, 101, 100, 91, 101, 101, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.95656111Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956585169Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956606673Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956628628Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956649991Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956676365Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"":"downstairs","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956698821Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956718722Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client"Wait for a query_work_queue command to finish before sending IO :"0","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956750307Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"1","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956771931Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"client":"2","":"downstairs","session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956793786Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956813847Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8 is now active with session: 48f2f16b-746e-42e3-8601-592e28afcd28","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.95683525Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956855131Z","hostname":"bmat-EVT22200007-0000589c","pid":1595,"session_id":"48f2f16b-746e-42e3-8601-592e28afcd28"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956875092Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} {"msg":"Activated sub_volume e5e3e8c5-2b1d-4b2f-81e0-b4771d0b36b8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:17.956893339Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 465 len 36864 data: 12 10 9 10 12 10 9 6 5 02/30 Write block 95 len 24576 data: 10 11 12 11 10 9 03/30 Read block 285 len 8192 04/30 Write block 164 len 8192 data: 12 10 05/30 Flush 06/30 Write block 304 len 4096 data: 6 07/30 Write block 539 len 20480 data: 7 7 7 7 6 08/30 Read block 210 len 8192 09/30 Write block 417 len 32768 data: 5 5 6 8 6 6 6 5 10/30 Write block 73 len 20480 data: 5 5 7 6 6 11/30 Read block 238 len 40960 12/30 Write block 346 len 12288 data: 9 9 8 13/30 Read block 562 len 8192 14/30 Read block 462 len 32768 15/30 Read block 272 len 16384 16/30 Read block 538 len 16384 17/30 Flush 18/30 Write block 124 len 20480 data: 7 5 6 6 6 19/30 Read block 234 len 8192 20/30 Read block 394 len 20480 21/30 Write block 191 len 36864 data: 8 9 7 9 10 9 9 8 6 22/30 Read block 53 len 20480 23/30 Write block 17 len 16384 data: 4 4 3 3 24/30 Write block 401 len 20480 data: 10 8 9 11 11 25/30 Read block 234 len 32768 26/30 Read block 355 len 12288 27/30 Write block 92 len 4096 data: 10 28/30 Write block 317 len 4096 data: 8 29/30 Write block 349 len 16384 data: 5 5 4 6 30/30 Read block 62 len 8192 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:17.988866535Z","hostname":"bmat-EVT22200007-0000589c","pid":1595} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 184 184 184 T T T 1 020-039 100 100 100 185 185 185 T T T 2 040-059 101 101 101 188 188 188 F F F 3 060-079 102 102 102 190 190 190 F F F 4 080-099 102 102 102 189 189 189 T T T 5 100-119 102 102 102 189 189 189 F F F 6 120-139 100 100 100 187 187 187 T T T 7 140-159 91 91 91 172 172 172 F F F 8 160-179 102 102 102 189 189 189 F F F 9 180-199 101 101 101 188 188 188 T T T 10 200-219 99 99 99 184 184 184 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 102 102 102 190 190 190 T T T 16 320-339 101 101 101 188 188 188 F F F 17 340-359 102 102 102 190 190 190 T T T 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 102 102 102 190 190 190 T T T 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 102 102 102 190 190 190 F F F 27 540-559 102 102 102 190 190 190 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 102, Max flush: 190 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 191 191 191 F F F 1 020-039 103 103 103 191 191 191 F F F 2 040-059 101 101 101 188 188 188 F F F 3 060-079 102 102 102 190 190 190 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 102 102 102 189 189 189 F F F 6 120-139 103 103 103 191 191 191 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 102 102 102 189 189 189 F F F 9 180-199 103 103 103 191 191 191 F F F 10 200-219 99 99 99 184 184 184 F F F 11 220-239 92 92 92 173 173 173 F F F 12 240-259 93 93 93 174 174 174 F F F 13 260-279 98 98 98 182 182 182 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 103 103 103 191 191 191 F F F 16 320-339 101 101 101 188 188 188 F F F 17 340-359 103 103 103 191 191 191 F F F 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 98 98 98 183 183 183 F F F 25 500-519 89 89 89 169 169 169 F F F 26 520-539 102 102 102 190 190 190 F F F 27 540-559 102 102 102 190 190 190 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 103, Max flush: 191 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.416918153Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:22.471484914Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.472008014Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.512237602Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.514148041Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.514203254Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.514226131Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.514250761Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"Crucible 77403414-e27e-467f-bf6b-12069f6a4f23 has session id: 015ca747-f4ac-4cf3-ac6e-47ac41ca951a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.515313656Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Upstairs opts: Upstairs UUID: 77403414-e27e-467f-bf6b-12069f6a4f23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.515367416Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Crucible stats registered with UUID: 77403414-e27e-467f-bf6b-12069f6a4f23","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.515397177Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:22.515417789Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516276035Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516363273Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"77403414-e27e-467f-bf6b-12069f6a4f23 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516383194Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516405079Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516461975Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516485072Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516509492Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516531668Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516555376Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516577171Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516600348Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.516623616Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33776)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.517139481Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45430)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.51720731Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49776)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.517242001Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.518159388Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.518215062Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.518854059Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.51954378Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.519581668Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.520258292Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.520431797Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.520560611Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 188, 190, 191, 189, 191, 172, 189, 191, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.520951771Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[0]R generation[0..12]: [103, 103, 101, 102, 103, 102, 103, 91, 102, 103, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.52098041Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521002665Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 188, 190, 191, 189, 191, 172, 189, 191, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521024299Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[1]R generation[0..12]: [103, 103, 101, 102, 103, 102, 103, 91, 102, 103, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521055152Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521077158Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 188, 190, 191, 189, 191, 172, 189, 191, 184, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521098411Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[2]R generation[0..12]: [103, 103, 101, 102, 103, 102, 103, 91, 102, 103, 99, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521119674Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521140948Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521160939Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.52118106Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.5212006Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521222154Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"":"downstairs","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521242516Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521260964Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"0","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521281385Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"1","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521301386Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"client":"2","":"downstairs","session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521321548Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521339945Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"77403414-e27e-467f-bf6b-12069f6a4f23 is now active with session: 015ca747-f4ac-4cf3-ac6e-47ac41ca951a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521363303Z","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.52138146ZWait for a query_work_queue command to finish before sending IO ","hostname":"bmat-EVT22200007-0000589c","pid":1603,"session_id":"015ca747-f4ac-4cf3-ac6e-47ac41ca951a"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521415029Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} {"msg":"Activated sub_volume 77403414-e27e-467f-bf6b-12069f6a4f23","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:22.521432454Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 16384 data: 13 14 12 11 02/30 Write block 262 len 40960 data: 9 9 8 10 11 10 11 11 11 10 03/30 Flush 04/30 Write block 514 len 8192 data: 10 10 05/30 Flush 06/30 Write block 108 len 28672 data: 10 9 11 10 11 13 11 07/30 Read block 543 len 28672 08/30 Write block 546 len 40960 data: 7 9 9 10 10 9 9 9 10 8 09/30 Write block 130 len 16384 data: 6 6 6 6 10/30 Write block 5 len 20480 data: 10 11 10 9 8 11/30 Read block 580 len 40960 12/30 Write block 47 len 16384 data: 10 7 7 8 13/30 Read block 356 len 16384 14/30 Write block 223 len 36864 data: 11 11 11 7 5 7 6 7 8 15/30 Write block 480 len 40960 data: 9 9 10 11 7 9 9 10 10 8 16/30 Write block 320 len 12288 data: 10 8 8 17/30 Read block 509 len 28672 18/30 Read block 231 len 36864 19/30 Write block 526 len 4096 data: 7 20/30 Read block 113 len 40960 21/30 Write block 181 len 20480 data: 7 4 4 5 7 22/30 Read block 171 len 24576 23/30 Write block 237 len 20480 data: 9 9 5 6 8 24/30 Write block 204 len 40960 data: 10 10 10 7 8 7 8 5 6 5 25/30 Read block 377 len 8192 26/30 Read block 152 len 36864 27/30 Read block 381 len 24576 28/30 Write block 222 len 16384 data: 11 12 12 12 29/30 Write block 554 len 20480 data: 11 9 10 10 9 30/30 Read block 68 len 36864 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 7 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 24 24 25 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:22.556125144Z","hostname":"bmat-EVT22200007-0000589c","pid":1603} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 191 191 191 T T T 1 020-039 103 103 103 191 191 191 F F F 2 040-059 101 101 101 188 188 188 T T T 3 060-079 102 102 102 190 190 190 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 102 102 102 189 189 189 T T T 6 120-139 103 103 103 191 191 191 T T T 7 140-159 91 91 91 172 172 172 F F F 8 160-179 104 104 104 192 192 192 F F F 9 180-199 103 103 103 191 191 191 T T T 10 200-219 99 99 99 184 184 184 T T T 11 220-239 92 92 92 173 173 173 T T T 12 240-259 93 93 93 174 174 174 T T T 13 260-279 104 104 104 192 192 192 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 103 103 103 191 191 191 F F F 16 320-339 101 101 101 188 188 188 T T T 17 340-359 103 103 103 191 191 191 F F F 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 98 98 98 183 183 183 T T T 25 500-519 104 104 104 193 193 193 F F F 26 520-539 102 102 102 190 190 190 T T T 27 540-559 102 102 102 190 190 190 T T T 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 104, Max flush: 193 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 194 194 194 F F F 1 020-039 103 103 103 191 191 191 F F F 2 040-059 105 105 105 194 194 194 F F F 3 060-079 102 102 102 190 190 190 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 105 105 105 194 194 194 F F F 6 120-139 105 105 105 194 194 194 F F F 7 140-159 91 91 91 172 172 172 F F F 8 160-179 104 104 104 192 192 192 F F F 9 180-199 105 105 105 194 194 194 F F F 10 200-219 105 105 105 194 194 194 F F F 11 220-239 105 105 105 194 194 194 F F F 12 240-259 105 105 105 194 194 194 F F F 13 260-279 104 104 104 192 192 192 F F F 14 280-299 87 87 87 164 164 164 F F F 15 300-319 103 103 103 191 191 191 F F F 16 320-339 105 105 105 194 194 194 F F F 17 340-359 103 103 103 191 191 191 F F F 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 105 105 105 194 194 194 F F F 27 540-559 105 105 105 194 194 194 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 98 98 98 182 182 182 F F F Max gen: 105, Max flush: 194 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.000610625Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.055763425Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.057537936Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.057612859Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.057642099Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.057670086Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"Crucible 02a8a036-4421-4a0d-91ba-e5ac89a01833 has session id: 9a4c0b0e-c579-4e80-84da-a8191ebeb326","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.058521839Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Upstairs opts: Upstairs UUID: 02a8a036-4421-4a0d-91ba-e5ac89a01833, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.058641764Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Crucible stats registered with UUID: 02a8a036-4421-4a0d-91ba-e5ac89a01833","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.058667587Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:27.0586836Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059329271Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059352909Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"02a8a036-4421-4a0d-91ba-e5ac89a01833 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059466752Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059486573Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059520993Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059538178Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059555293Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059571426Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059588361Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059603793Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059686592Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.059710761Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"ds_connection connected from Ok(127.0.0.1:37661)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.060058042Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"ds_connection connected from Ok(127.0.0.1:46664)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.060129809Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:27.060155702Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:27.060177136Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:27.060198149Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.060213881Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.060818037Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.060967503Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.061522388Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.061673377Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.062179082Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:27.062298897Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.061380793Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"ds_connection connected from Ok(127.0.0.1:49936)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.061502652Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.063134982Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.064582313Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[0]R flush_numbers[0..12]: [194, 191, 194, 190, 191, 194, 194, 172, 192, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065805906Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[0]R generation[0..12]: [105, 103, 105, 102, 103, 105, 105, 91, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065838754Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065859967Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[1]R flush_numbers[0..12]: [194, 191, 194, 190, 191, 194, 194, 172, 192, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065880128Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[1]R generation[0..12]: [105, 103, 105, 102, 103, 105, 105, 91, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065910551Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065931062Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[2]R flush_numbers[0..12]: [194, 191, 194, 190, 191, 194, 194, 172, 192, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065950633Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[2]R generation[0..12]: [105, 103, 105, 102, 103, 105, 105, 91, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065970213Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.065990374Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066009272Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066028181Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066046669Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066065147Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"":"downstairs","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066083625Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066100339Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"0","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066118746Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"1","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066137214Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"client":"2","":"downstairs","session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066156253Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066173438Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"02a8a036-4421-4a0d-91ba-e5ac89a01833 is now active with session: 9a4c0b0e-c579-4e80-84da-a8191ebeb326","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066195233Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066212148Z","hostname":"bmat-EVT22200007-0000589c","pid":1611,"session_id":"9a4c0b0e-c579-4e80-84da-a8191ebeb326"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066229583Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} {"msg":"Activated sub_volume 02a8a036-4421-4a0d-91ba-e5ac89a01833","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:37.066354679Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 534 len 24576 data: 5 6 8 7 8 8 02/30 Write block 134 len 36864 data: 7 9 9 10 11 13 12 11 9 03/30 Read block 401 len 40960 04/30 Write block 114 len 20480 data: 12 12 13 11 10 05/30 Read block 181 len 20480 06/30 Read block 82 len 40960 07/30 Read block 270 len 4096 08/30 Write block 582 len 32768 data: 7 9 9 8 7 6 6 7 09/30 Read block 1 len 20480 10/30 Read block 405 len 32768 11/30 Read block 363 len 24576 12/30 Write block 526 len 8192 data: 8 9 13/30 Read block 374 len 28672 14/30 Write block 167 len 8192 data: 14 15 15/30 Read block 321 len 40960 16/30 Flush 17/30 Write block 318 len 32768 data: 9 10 11 9 9 6 7 7 18/30 Read block 388 len 32768 19/30 Write block 113 len 16384 data: 14 13 13 14 20/30 Write block 220 len 28672 data: 8 8 12 13 13 13 8 21/30 Read block 380 len 12288 22/30 Read block 17 len 8192 23/30 Read block 289 len 36864 24/30 Write block 196 len 20480 data: 10 10 9 7 9 25/30 Write block 283 len 20480 data: 6 4 4 5 4 26/30 Write block 542 len 8192 data: 8 7 27/30 Read block 96 len 12288 28/30 Read block 545 len 40960 29/30 Write block 27 len 20480 data: 8 7 6 7 7 30/30 Write block 347 len 36864 data: 10 9 6 6 5 7 5 5 6 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:37.099937578Z","hostname":"bmat-EVT22200007-0000589c","pid":1611} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 194 194 194 F F F 1 020-039 103 103 103 191 191 191 T T T 2 040-059 105 105 105 194 194 194 F F F 3 060-079 102 102 102 190 190 190 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 106 106 106 195 195 195 T T T 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 105 105 105 194 194 194 T T T 10 200-219 105 105 105 194 194 194 T T T 11 220-239 105 105 105 194 194 194 T T T 12 240-259 105 105 105 194 194 194 F F F 13 260-279 104 104 104 192 192 192 F F F 14 280-299 87 87 87 164 164 164 T T T 15 300-319 103 103 103 191 191 191 T T T 16 320-339 105 105 105 194 194 194 T T T 17 340-359 103 103 103 191 191 191 T T T 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 106 106 106 195 195 195 F F F 27 540-559 105 105 105 194 194 194 T T T 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 106, Max flush: 195 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 194 194 194 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 105 105 105 194 194 194 F F F 3 060-079 102 102 102 190 190 190 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 107 107 107 196 196 196 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 107 107 107 196 196 196 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 105 105 105 194 194 194 F F F 13 260-279 104 104 104 192 192 192 F F F 14 280-299 107 107 107 196 196 196 F F F 15 300-319 107 107 107 196 196 196 F F F 16 320-339 107 107 107 196 196 196 F F F 17 340-359 107 107 107 196 196 196 F F F 18 360-379 101 101 101 188 188 188 F F F 19 380-399 101 101 101 188 188 188 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 106 106 106 195 195 195 F F F 27 540-559 107 107 107 196 196 196 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 107, Max flush: 196 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.468545879Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.523833615Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.524871899Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.525017377Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.525037999Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.525056267Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"Crucible ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c has session id: 55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.525978263Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Upstairs opts: Upstairs UUID: ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.526132438Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Crucible stats registered with UUID: ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.526170506Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:41.526199696Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.526947847Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527004603Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527030416Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527049615Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527087512Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.52710584Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527132655Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527150441Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527169691Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527187046Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527270046Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527295548Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:42251)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527386334Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:43337)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527494856Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:41.527525519Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:41.527546061Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:41.527807165Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.527839651Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.528796299Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.528862003Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.529628832Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.529766033Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.530407966Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:41.530434831Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.530173239Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"ds_connection connected from Ok(127.0.0.1:51059)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.530611606Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.531843166Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.532649386Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[0]R flush_numbers[0..12]: [194, 196, 194, 190, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533486789Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[0]R generation[0..12]: [105, 107, 105, 102, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.53351679Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533536721Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[1]R flush_numbers[0..12]: [194, 196, 194, 190, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533554888Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[1]R generation[0..12]: [105, 107, 105, 102, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533572785Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533591212Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[2]R flush_numbers[0..12]: [194, 196, 194, 190, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533618709Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[2]R generation[0..12]: [105, 107, 105, 102, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533636475Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533654452Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533670966Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.53368797Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533704414Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533720707Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"":"downstairs","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533737402Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533752563Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"0","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533769076Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"1","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.53378567Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"client":"2","":"downstairs","session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533802655Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533818177Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c is now active with session: 55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.53383442Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533849751Z","hostname":"bmat-EVT22200007-0000589c","pid":1619,"session_id":"55dbba9a-8b9b-4d0d-b8f4-4a44ee857c44"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.533865454Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} {"msg":"Activated sub_volume ce4679ee-c7de-4ca6-9c7b-bbaa63b1418c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:51.53401991Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 24576 data: 6 6 6 6 8 7 02/30 Read block 197 len 24576 03/30 Read block 375 len 32768 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 363 len 8192 data: 8 10 08/30 Write block 282 len 12288 data: 7 7 5 09/30 Read block 265 len 8192 10/30 Write block 5 len 12288 data: 11 12 11 11/30 Read block 566 len 4096 12/30 Flush 13/30 Write block 386 len 40960 data: 9 12 11 10 11 16 14 13 13 13 14/30 Flush 15/30 Write block 346 len 28672 data: 10 11 10 7 7 6 8 16/30 Read block 581 len 4096 17/30 Flush 18/30 Flush 19/30 Write block 287 len 36864 data: 5 4 5 5 6 6 7 9 10 20/30 Write block 274 len 24576 data: 8 8 7 7 8 9 21/30 Flush 22/30 Write block 276 len 32768 data: 8 8 9 10 10 8 8 8 23/30 Read block 14 len 4096 24/30 Read block 310 len 28672 25/30 Read block 539 len 40960 26/30 Read block 32 len 36864 27/30 Write block 390 len 32768 data: 12 17 15 14 14 14 14 8 28/30 Write block 310 len 28672 data: 6 6 6 6 7 7 8 29/30 Write block 326 len 28672 data: 5 7 7 7 10 11 11 30/30 Read block 521 len 24576 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:51.566991177Z","hostname":"bmat-EVT22200007-0000589c","pid":1619} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 105 105 105 194 194 194 F F F 3 060-079 108 108 108 197 197 197 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 107 107 107 196 196 196 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 107 107 107 196 196 196 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 105 105 105 194 194 194 F F F 13 260-279 108 108 108 204 204 204 T T T 14 280-299 108 108 108 204 204 204 T T T 15 300-319 107 107 107 196 196 196 T T T 16 320-339 107 107 107 196 196 196 T T T 17 340-359 108 108 108 202 202 202 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 108 108 108 201 201 201 T T T 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 106 106 106 195 195 195 F F F 27 540-559 107 107 107 196 196 196 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 108, Max flush: 204 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 105 105 105 194 194 194 F F F 3 060-079 108 108 108 197 197 197 F F F 4 080-099 103 103 103 191 191 191 F F F 5 100-119 107 107 107 196 196 196 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 106 106 106 195 195 195 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 107 107 107 196 196 196 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 105 105 105 194 194 194 F F F 13 260-279 109 109 109 205 205 205 F F F 14 280-299 109 109 109 205 205 205 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 109 109 109 205 205 205 F F F 17 340-359 108 108 108 202 202 202 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 103 103 103 191 191 191 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 106 106 106 195 195 195 F F F 27 540-559 107 107 107 196 196 196 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 109, Max flush: 205 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.900943122Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:55.953273077Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.953669929Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.990071014Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.990746496Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.990790646Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.990815136Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.990833524Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"Crucible 37de85a6-013e-4719-9dc3-aac1c4529ce5 has session id: 7195bf1d-9b50-4038-afe4-a3ec36b8deee","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.991844793Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Upstairs opts: Upstairs UUID: 37de85a6-013e-4719-9dc3-aac1c4529ce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.992020482Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Crucible stats registered with UUID: 37de85a6-013e-4719-9dc3-aac1c4529ce5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.99204373Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:55.992059272Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.992789877Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.992819076Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"37de85a6-013e-4719-9dc3-aac1c4529ce5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.992946888Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.99296735Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993039116Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993057424Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.99307503Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993090722Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993115262Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993131125Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993147799Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993235949Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"ds_connection connected from Ok(127.0.0.1:54543)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993433914Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"ds_connection connected from Ok(127.0.0.1:49393)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993654385Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"ds_connection connected from Ok(127.0.0.1:65378)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.993786165Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.994902479Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.994967161Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.995139524Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.995628554Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.995664588Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.996101271Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.996319538Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.996419903Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[0]R flush_numbers[0..12]: [200, 196, 194, 197, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997075906Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[0]R generation[0..12]: [108, 107, 105, 108, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997103091Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997123423Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[1]R flush_numbers[0..12]: [200, 196, 194, 197, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997140107Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[1]R generation[0..12]: [108, 107, 105, 108, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997156871Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997173635Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[2]R flush_numbers[0..12]: [200, 196, 194, 197, 191, 196, 195, 195, 195, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997189989Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[2]R generation[0..12]: [108, 107, 105, 108, 103, 107, 106, 106, 106, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997206002Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997222445Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997237857Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997253449Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.99726857Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997284031Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"":"downstairs","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997299683Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997320155Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"0","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997336699Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"1","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997352251Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"client":"2","":"downstairs","session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997368564Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997382854Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"37de85a6-013e-4719-9dc3-aac1c4529ce5 is now active with session: 7195bf1d-9b50-4038-afe4-a3ec36b8deee","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997397954Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997411823Z","hostname":"bmat-EVT22200007-0000589c","pid":1627,"session_id":"7195bf1d-9b50-4038-afe4-a3ec36b8deee"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.997426423Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} {"msg":"Activated sub_volume 37de85a6-013e-4719-9dc3-aac1c4529ce5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:07:55.99743967Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 12288 data: 11 9 8 02/30 Read block 337 len 12288 03/30 Read block 277 len 28672 04/30 Flush 05/30 Write block 320 len 8192 data: 12 10 06/30 Read block 138 len 4096 07/30 Read block 456 len 20480 08/30 Write block 338 len 20480 data: 9 10 11 11 14 09/30 Read block 289 len 20480 10/30 Write block 156 len 20480 data: 6 5 7 11 13 11/30 Read block 85 len 8192 12/30 Read block 493 len 32768 13/30 Read block 36 len 4096 14/30 Read block 8 len 16384 15/30 Read block 38 len 32768 16/30 Write block 60 len 12288 data: 12 10 7 17/30 Write block 253 len 20480 data: 8 7 10 11 13 18/30 Flush 19/30 Write block 82 len 36864 data: 10 9 9 9 7 7 7 8 10 20/30 Write block 341 len 8192 data: 12 15 21/30 Read block 129 len 4096 22/30 Write block 401 len 16384 data: 11 9 10 12 23/30 Write block 330 len 40960 data: 11 12 12 10 10 8 9 9 10 11 24/30 Read block 462 len 32768 25/30 Read block 98 len 8192 26/30 Write block 533 len 32768 data: 5 6 7 9 8 9 9 8 27/30 Read block 451 len 4096 28/30 Write block 57 len 24576 data: 11 10 10 13 11 8 29/30 Read block 343 len 12288 30/30 Read block 131 len 20480 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:07:56.025032163Z","hostname":"bmat-EVT22200007-0000589c","pid":1627} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 105 105 105 194 194 194 T T T 3 060-079 110 110 110 207 207 207 T T T 4 080-099 110 110 110 206 206 206 T T T 5 100-119 107 107 107 196 196 196 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 110 110 110 207 207 207 F F F 8 160-179 110 110 110 207 207 207 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 107 107 107 196 196 196 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 110 110 110 207 207 207 F F F 13 260-279 109 109 109 205 205 205 F F F 14 280-299 109 109 109 205 205 205 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 110 110 110 207 207 207 T T T 17 340-359 110 110 110 207 207 207 T T T 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 103 103 103 191 191 191 T T T 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 106 106 106 195 195 195 T T T 27 540-559 107 107 107 196 196 196 T T T 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 110, Max flush: 207 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 111 111 111 208 208 208 F F F 3 060-079 111 111 111 208 208 208 F F F 4 080-099 111 111 111 208 208 208 F F F 5 100-119 107 107 107 196 196 196 F F F 6 120-139 106 106 106 195 195 195 F F F 7 140-159 110 110 110 207 207 207 F F F 8 160-179 110 110 110 207 207 207 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 107 107 107 196 196 196 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 110 110 110 207 207 207 F F F 13 260-279 109 109 109 205 205 205 F F F 14 280-299 109 109 109 205 205 205 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 111 111 111 208 208 208 F F F 17 340-359 111 111 111 208 208 208 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 111 111 111 208 208 208 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 111 111 111 208 208 208 F F F 27 540-559 111 111 111 208 208 208 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 111, Max flush: 208 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.407142377Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.460880172Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.461786075Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46183728Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.461856209Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.461873935Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"Crucible 210197b2-f946-4a9c-a6f9-164da51c4053 has session id: 9ce2c048-2c04-47e3-8a49-13895cc83593","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.462876536Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Upstairs opts: Upstairs UUID: 210197b2-f946-4a9c-a6f9-164da51c4053, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.462929765Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Crucible stats registered with UUID: 210197b2-f946-4a9c-a6f9-164da51c4053","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.462949456Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:00.462965148Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.463943229Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.463980355Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"210197b2-f946-4a9c-a6f9-164da51c4053 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464081061Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46410048Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464136614Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464322044Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464822167Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464919626Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464939327Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46495549Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.464973326Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.465050965Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"ds_connection connected from Ok(127.0.0.1:53976)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.465334064Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"ds_connection connected from Ok(127.0.0.1:63094)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.465465944Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"ds_connection connected from Ok(127.0.0.1:59717)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.465493501Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46551295Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.465531097Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46619207Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.466408072Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.466447552Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.467256958Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.467488021Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.467523664Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[0]R flush_numbers[0..12]: [200, 196, 208, 208, 208, 196, 195, 207, 207, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468463607Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[0]R generation[0..12]: [108, 107, 111, 111, 111, 107, 106, 110, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46848918Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468508589Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[1]R flush_numbers[0..12]: [200, 196, 208, 208, 208, 196, 195, 207, 207, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468525825Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[1]R generation[0..12]: [108, 107, 111, 111, 111, 107, 106, 110, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468542489Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468559263Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[2]R flush_numbers[0..12]: [200, 196, 208, 208, 208, 196, 195, 207, 207, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468575797Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[2]R generation[0..12]: [108, 107, 111, 111, 111, 107, 106, 110, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468598914Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468616861Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468632713Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468648235Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468663506Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468679389Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"":"downstairs","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468695041Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46870933Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"0","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468725393Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"1","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468741005Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"client":"2","":"downstairs","session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468756947Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.468771938Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"210197b2-f946-4a9c-a6f9-164da51c4053 is now active with session: 9ce2c048-2c04-47e3-8a49-13895cc83593","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46878732Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46880203Z","hostname":"bmat-EVT22200007-0000589c","pid":1635,"session_id":"9ce2c048-2c04-47e3-8a49-13895cc83593"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:00.46881699Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} {"msg":"Activated sub_volume 210197b2-f946-4a9c-a6f9-164da51c4053","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:08:00.468830678Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 24576 data: 12 14 14 12 9 9 02/30 Write block 90 len 4096 data: 11 03/30 Read block 549 len 8192 04/30 Write block 548 len 8192 data: 10 11 05/30 Read block 103 len 16384 06/30 Read block 356 len 4096 07/30 Read block 252 len 12288 08/30 Read block 263 len 32768 09/30 Write block 66 len 28672 data: 8 9 8 8 7 7 7 10/30 Write block 218 len 4096 data: 7 11/30 Read block 562 len 32768 12/30 Read block 591 len 16384 13/30 Write block 329 len 4096 data: 8 14/30 Read block 475 len 28672 15/30 Read block 452 len 40960 16/30 Write block 400 len 12288 data: 10 12 10 17/30 Flush 18/30 Flush 19/30 Write block 333 len 20480 data: 11 11 9 10 10 20/30 Read block 362 len 16384 21/30 Write block 125 len 16384 data: 6 7 7 7 22/30 Read block 348 len 4096 23/30 Read block 333 len 16384 24/30 Flush 25/30 Write block 528 len 4096 data: 10 26/30 Read block 286 len 40960 27/30 Read block 223 len 8192 28/30 Read block 463 len 12288 29/30 Write block 104 len 40960 data: 9 10 10 9 11 10 12 11 12 15 30/30 Read block 266 len 8192 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:00.496939979Z","hostname":"bmat-EVT22200007-0000589c","pid":1635} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 111 111 111 208 208 208 F F F 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 107 107 107 196 196 196 T T T 6 120-139 112 112 112 211 211 211 F F F 7 140-159 112 112 112 209 209 209 F F F 8 160-179 110 110 110 207 207 207 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 110 110 110 207 207 207 F F F 13 260-279 109 109 109 205 205 205 F F F 14 280-299 109 109 109 205 205 205 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 112 112 112 211 211 211 F F F 17 340-359 111 111 111 208 208 208 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 111 111 111 208 208 208 T T T 27 540-559 112 112 112 209 209 209 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 112, Max flush: 211 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 F F F 2 040-059 111 111 111 208 208 208 F F F 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 112 112 112 211 211 211 F F F 7 140-159 112 112 112 209 209 209 F F F 8 160-179 110 110 110 207 207 207 F F F 9 180-199 107 107 107 196 196 196 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 107 107 107 196 196 196 F F F 12 240-259 110 110 110 207 207 207 F F F 13 260-279 109 109 109 205 205 205 F F F 14 280-299 109 109 109 205 205 205 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 112 112 112 211 211 211 F F F 17 340-359 111 111 111 208 208 208 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 F F F 28 560-579 100 100 100 185 185 185 F F F 29 580-599 106 106 106 195 195 195 F F F Max gen: 113, Max flush: 212 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.796934998Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.85310051Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.854219951Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.854310255Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.854342812Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.854373765Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"Crucible 683f9a43-7ca1-435e-b925-60dde03eaf4c has session id: ad50722f-18f6-475c-84f8-18af3fcbf142","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.855673424Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Upstairs opts: Upstairs UUID: 683f9a43-7ca1-435e-b925-60dde03eaf4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.855731343Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Crucible stats registered with UUID: 683f9a43-7ca1-435e-b925-60dde03eaf4c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.855753077Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:04.855768549Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856464122Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856493192Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"683f9a43-7ca1-435e-b925-60dde03eaf4c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856621634Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856641826Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856671196Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.856688271Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.85781184Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.85785131Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.857879899Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.857901633Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.857920762Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.85793898Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"ds_connection connected from Ok(127.0.0.1:42414)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858303956Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"ds_connection connected from Ok(127.0.0.1:64766)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858341864Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"ds_connection connected from Ok(127.0.0.1:32977)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858543466Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858602607Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858622358Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858640014Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858710979Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.858898874Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.859539564Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.859661053Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.85968408Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[0]R flush_numbers[0..12]: [200, 196, 208, 209, 209, 212, 211, 209, 207, 196, 209, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860506112Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[0]R generation[0..12]: [108, 107, 111, 112, 112, 113, 112, 112, 110, 107, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.8605294Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860547617Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[1]R flush_numbers[0..12]: [200, 196, 208, 209, 209, 212, 211, 209, 207, 196, 209, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860571305Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[1]R generation[0..12]: [108, 107, 111, 112, 112, 113, 112, 112, 110, 107, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860589032Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860606818Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[2]R flush_numbers[0..12]: [200, 196, 208, 209, 209, 212, 211, 209, 207, 196, 209, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860623863Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[2]R generation[0..12]: [108, 107, 111, 112, 112, 113, 112, 112, 110, 107, 112, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860641168Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860658714Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860674817Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860691641Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860707885Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860723857Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"":"downstairs","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860740441Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860755322Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"0","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-23T06:08:04.860772046Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"1","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860802498Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"client":"2","":"downstairs","session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860819032Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.86083762Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"683f9a43-7ca1-435e-b925-60dde03eaf4c is now active with session: ad50722f-18f6-475c-84f8-18af3fcbf142","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860853984Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860868944Z","hostname":"bmat-EVT22200007-0000589c","pid":1643,"session_id":"ad50722f-18f6-475c-84f8-18af3fcbf142"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860884326Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} {"msg":"Activated sub_volume 683f9a43-7ca1-435e-b925-60dde03eaf4c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:04.860898815Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 8192 data: 7 6 02/30 Read block 382 len 32768 03/30 Read block 488 len 12288 04/30 Read block 415 len 28672 05/30 Read block 578 len 24576 06/30 Read block 1 len 4096 07/30 Write block 226 len 8192 data: 9 6 08/30 Read block 432 len 24576 09/30 Write block 333 len 40960 data: 12 12 10 11 11 11 12 12 13 16 10/30 Read block 48 len 8192 11/30 Write block 355 len 20480 data: 7 9 10 10 12 12/30 Write block 186 len 16384 data: 8 8 7 6 13/30 Write block 444 len 20480 data: 9 8 10 9 10 14/30 Write block 586 len 24576 data: 8 7 7 8 6 7 15/30 Write block 159 len 24576 data: 12 14 11 11 13 13 16/30 Write block 125 len 40960 data: 7 8 8 8 7 7 7 7 7 8 17/30 Write block 452 len 12288 data: 6 6 5 18/30 Read block 234 len 24576 19/30 Read block 385 len 12288 20/30 Read block 55 len 12288 21/30 Write block 190 len 36864 data: 8 9 10 8 10 11 11 11 10 22/30 Read block 407 len 16384 23/30 Write block 49 len 24576 data: 8 9 8 12 13 12 24/30 Write block 25 len 36864 data: 8 9 9 8 7 8 8 9 7 25/30 Write block 579 len 36864 data: 8 8 8 8 10 10 9 9 8 26/30 Read block 543 len 12288 27/30 Write block 287 len 20480 data: 6 5 6 6 7 28/30 Write block 53 len 12288 data: 14 13 11 29/30 Write block 255 len 28672 data: 11 12 14 11 10 7 8 30/30 Read block 596 len 16384 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:04.887922084Z","hostname":"bmat-EVT22200007-0000589c","pid":1643} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 196 196 196 T T T 2 040-059 111 111 111 208 208 208 T T T 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 112 112 112 211 211 211 T T T 7 140-159 112 112 112 209 209 209 T T T 8 160-179 110 110 110 207 207 207 T T T 9 180-199 107 107 107 196 196 196 T T T 10 200-219 112 112 112 209 209 209 F F F 11 220-239 107 107 107 196 196 196 T T T 12 240-259 110 110 110 207 207 207 T T T 13 260-279 109 109 109 205 205 205 T T T 14 280-299 109 109 109 205 205 205 T T T 15 300-319 109 109 109 205 205 205 F F F 16 320-339 112 112 112 211 211 211 T T T 17 340-359 111 111 111 208 208 208 T T T 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 101 101 101 188 188 188 T T T 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 F F F 28 560-579 100 100 100 185 185 185 T T T 29 580-599 106 106 106 195 195 195 T T T Max gen: 113, Max flush: 212 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 213 213 213 F F F 2 040-059 114 114 114 213 213 213 F F F 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 114 114 114 213 213 213 F F F 7 140-159 114 114 114 213 213 213 F F F 8 160-179 114 114 114 213 213 213 F F F 9 180-199 114 114 114 213 213 213 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 114 114 114 213 213 213 F F F 12 240-259 114 114 114 213 213 213 F F F 13 260-279 114 114 114 213 213 213 F F F 14 280-299 114 114 114 213 213 213 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 114 114 114 213 213 213 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 205 205 205 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 102 102 102 190 190 190 F F F 22 440-459 114 114 114 213 213 213 F F F 23 460-479 102 102 102 189 189 189 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 104 104 104 193 193 193 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 114 114 114 213 213 213 F F F Max gen: 114, Max flush: 213 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.283047538Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.338036643Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.340290274Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.340384407Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34040565Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.340424138Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"Crucible 94cc7832-8878-4ba1-8b44-45228ca2aab8 has session id: aecd739c-240a-40f9-b44c-b28c81aecd4e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.341297886Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Upstairs opts: Upstairs UUID: 94cc7832-8878-4ba1-8b44-45228ca2aab8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.341347607Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Crucible stats registered with UUID: 94cc7832-8878-4ba1-8b44-45228ca2aab8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.341366897Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:09.341382188Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342248962Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342281678Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"94cc7832-8878-4ba1-8b44-45228ca2aab8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34240452Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342425062Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342454843Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342471577Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342488652Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342504735Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342522361Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342538133Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.342555088Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34299131Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57191)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.343027765Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50058)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.343049189Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45143)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.343066103Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.343927836Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.344179131Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.344213571Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.344808378Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.345373925Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.345570116Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34567445Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34583105Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[0]R flush_numbers[0..12]: [200, 213, 213, 209, 209, 212, 213, 213, 213, 213, 209, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346620265Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[0]R generation[0..12]: [108, 114, 114, 112, 112, 113, 114, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346663093Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346682322Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[1]R flush_numbers[0..12]: [200, 213, 213, 209, 209, 212, 213, 213, 213, 213, 209, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346699527Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[1]R generation[0..12]: [108, 114, 114, 112, 112, 113, 114, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34671553Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346732595Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[2]R flush_numbers[0..12]: [200, 213, 213, 209, 209, 212, 213, 213, 213, 213, 209, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346749089Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[2]R generation[0..12]: [108, 114, 114, 112, 112, 113, 114, 114, 114, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346775302Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346792808Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.34680846Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346824272Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.346839494Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:08:09.346854604Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"":"downstairs","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347069123Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347088583Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"0","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347105999Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"1","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347122633Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"client":"2","":"downstairs","session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347139016Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347153576Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"94cc7832-8878-4ba1-8b44-45228ca2aab8 is now active with session: aecd739c-240a-40f9-b44c-b28c81aecd4e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347169038Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347183367Z","hostname":"bmat-EVT22200007-0000589c","pid":1651,"session_id":"aecd739c-240a-40f9-b44c-b28c81aecd4e"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347198137Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} {"msg":"Activated sub_volume 94cc7832-8878-4ba1-8b44-45228ca2aab8","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:09.347211565Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 16384 data: 12 8 8 9 02/30 Write block 169 len 8192 data: 13 12 03/30 Write block 463 len 24576 data: 12 12 13 11 10 11 04/30 Write block 91 len 4096 data: 9 05/30 Write block 429 len 40960 data: 7 8 9 11 12 10 11 11 9 10 06/30 Write block 86 len 8192 data: 8 8 07/30 Read block 97 len 8192 08/30 Write block 510 len 12288 data: 11 10 9 09/30 Write block 591 len 36864 data: 8 6 5 6 5 5 7 4 3 10/30 Write block 289 len 32768 data: 7 7 8 7 8 10 11 12 11/30 Read block 39 len 12288 12/30 Write block 72 len 4096 data: 8 13/30 Read block 537 len 28672 14/30 Read block 329 len 16384 15/30 Read block 334 len 24576 16/30 Write block 52 len 24576 data: 13 15 14 12 10 12 17/30 Flush 18/30 Write block 374 len 36864 data: 18 16 16 13 14 15 12 12 8 19/30 Write block 190 len 24576 data: 9 10 11 9 11 12 20/30 Write block 428 len 24576 data: 6 8 9 10 12 13 21/30 Flush 22/30 Flush 23/30 Write block 157 len 8192 data: 6 8 24/30 Read block 523 len 40960 25/30 Write block 163 len 8192 data: 14 14 26/30 Write block 166 len 12288 data: 9 15 16 27/30 Read block 261 len 32768 28/30 Read block 368 len 12288 29/30 Write block 130 len 28672 data: 8 8 8 8 9 10 10 30/30 Read block 298 len 4096 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:09.37122265Z","hostname":"bmat-EVT22200007-0000589c","pid":1651} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 213 213 213 F F F 2 040-059 116 116 116 214 214 214 F F F 3 060-079 116 116 116 214 214 214 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 114 114 114 213 213 213 T T T 7 140-159 114 114 114 213 213 213 T T T 8 160-179 116 116 116 214 214 214 T T T 9 180-199 116 116 116 215 215 215 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 116 116 116 214 214 214 F F F 12 240-259 114 114 114 213 213 213 F F F 13 260-279 114 114 114 213 213 213 F F F 14 280-299 116 116 116 214 214 214 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 114 114 114 213 213 213 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 116 116 116 215 215 215 F F F 19 380-399 116 116 116 215 215 215 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 114 114 114 213 213 213 F F F 23 460-479 116 116 116 214 214 214 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 116 116 116 214 214 214 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 116 116 116 214 214 214 F F F Max gen: 116, Max flush: 215 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 213 213 213 F F F 2 040-059 116 116 116 214 214 214 F F F 3 060-079 116 116 116 214 214 214 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 117 117 117 216 216 216 F F F 7 140-159 117 117 117 216 216 216 F F F 8 160-179 117 117 117 216 216 216 F F F 9 180-199 116 116 116 215 215 215 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 116 116 116 214 214 214 F F F 12 240-259 114 114 114 213 213 213 F F F 13 260-279 114 114 114 213 213 213 F F F 14 280-299 116 116 116 214 214 214 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 114 114 114 213 213 213 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 116 116 116 215 215 215 F F F 19 380-399 116 116 116 215 215 215 F F F 20 400-419 112 112 112 209 209 209 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 114 114 114 213 213 213 F F F 23 460-479 116 116 116 214 214 214 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 116 116 116 214 214 214 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 116 116 116 214 214 214 F F F Max gen: 117, Max flush: 216 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.674710709Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:13.726017139Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.726432239Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.763579961Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.765500029Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.765573299Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.765615375Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.765649235Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"Crucible d00ebe86-feb4-4a37-9604-3572d41cb807 has session id: 0b804e5e-88db-4cbd-8a37-cec0aeee5be6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.766540909Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Upstairs opts: Upstairs UUID: d00ebe86-feb4-4a37-9604-3572d41cb807, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.766608668Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Crucible stats registered with UUID: d00ebe86-feb4-4a37-9604-3572d41cb807","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.766629851Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:13.766645413Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767410278Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767440419Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"d00ebe86-feb4-4a37-9604-3572d41cb807 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767561858Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.76758224Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767614205Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.76763138Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767648716Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767664458Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767681342Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767706273Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.76772421Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767740584Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37145)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767936344Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41900)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.767976146Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42469)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.768250747Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.769116519Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.769150458Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.769231704Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.769938961Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.770113779Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.770142418Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.770893224Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.770921602Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[0]R flush_numbers[0..12]: [200, 213, 214, 214, 214, 212, 216, 216, 216, 215, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771033591Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[0]R generation[0..12]: [108, 114, 116, 116, 116, 113, 117, 117, 117, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771065566Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771084104Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[1]R flush_numbers[0..12]: [200, 213, 214, 214, 214, 212, 216, 216, 216, 215, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771101249Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[1]R generation[0..12]: [108, 114, 116, 116, 116, 113, 117, 117, 117, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771119116Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771136932Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[2]R flush_numbers[0..12]: [200, 213, 214, 214, 214, 212, 216, 216, 216, 215, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771154107Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[2]R generation[0..12]: [108, 114, 116, 116, 116, 113, 117, 117, 117, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771171363Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771188929Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771204671Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771220634Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771236225Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771251697Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"":"downstairs","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.77126777Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771282801Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"0","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771303223Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"1","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771319756Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"client":"2","":"downstairs","session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.77133599Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.7713509Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"d00ebe86-feb4-4a37-9604-3572d41cb807 is now active with session: 0b804e5e-88db-4cbd-8a37-cec0aeee5be6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771366662Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771381252Z","hostname":"bmat-EVT22200007-0000589c","pid":1659,"session_id":"0b804e5e-88db-4cbd-8a37-cec0aeee5be6"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771396674Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} {"msg":"Activated sub_volume d00ebe86-feb4-4a37-9604-3572d41cb807","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:13.771410813Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 247 len 40960 data: 12 9 10 10 10 8 9 8 12 13 02/30 Read block 204 len 20480 03/30 Write block 235 len 20480 data: 9 10 10 10 6 04/30 Write block 450 len 40960 data: 8 7 7 7 6 6 9 8 8 8 05/30 Read block 8 len 4096 06/30 Write block 333 len 8192 data: 13 13 07/30 Write block 586 len 28672 data: 10 9 8 9 7 9 7 08/30 Write block 289 len 28672 data: 8 8 9 8 9 11 12 09/30 Write block 143 len 8192 data: 13 15 10/30 Flush 11/30 Write block 405 len 4096 data: 12 12/30 Read block 355 len 4096 13/30 Read block 10 len 32768 14/30 Write block 448 len 12288 data: 11 9 9 15/30 Write block 7 len 4096 data: 12 16/30 Read block 527 len 40960 17/30 Write block 379 len 16384 data: 16 13 13 9 18/30 Read block 267 len 24576 19/30 Read block 357 len 8192 20/30 Read block 504 len 20480 21/30 Read block 305 len 8192 22/30 Write block 445 len 36864 data: 9 11 10 12 10 10 8 8 8 23/30 Read block 11 len 12288 24/30 Read block 577 len 16384 25/30 Write block 6 len 28672 data: 13 13 10 9 6 5 8 26/30 Write block 406 len 12288 data: 10 10 12 27/30 Write block 279 len 28672 data: 11 11 9 9 9 6 5 28/30 Write block 541 len 12288 data: 8 9 8 29/30 Read block 329 len 8192 30/30 Read block 584 len 28672 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 20 20 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:13.797905679Z","hostname":"bmat-EVT22200007-0000589c","pid":1659} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 T T T 1 020-039 114 114 114 213 213 213 F F F 2 040-059 116 116 116 214 214 214 F F F 3 060-079 116 116 116 214 214 214 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 117 117 117 216 216 216 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 117 117 117 216 216 216 F F F 9 180-199 116 116 116 215 215 215 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 118 118 118 217 217 217 F F F 13 260-279 114 114 114 213 213 213 T T T 14 280-299 118 118 118 217 217 217 T T T 15 300-319 109 109 109 205 205 205 F F F 16 320-339 118 118 118 217 217 217 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 116 116 116 215 215 215 T T T 19 380-399 116 116 116 215 215 215 T T T 20 400-419 112 112 112 209 209 209 T T T 21 420-439 116 116 116 215 215 215 F F F 22 440-459 118 118 118 217 217 217 T T T 23 460-479 116 116 116 214 214 214 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 116 116 116 214 214 214 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 112 112 112 209 209 209 T T T 28 560-579 114 114 114 213 213 213 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 118, Max flush: 217 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 218 218 218 F F F 1 020-039 114 114 114 213 213 213 F F F 2 040-059 116 116 116 214 214 214 F F F 3 060-079 116 116 116 214 214 214 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 117 117 117 216 216 216 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 117 117 117 216 216 216 F F F 9 180-199 116 116 116 215 215 215 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 118 118 118 217 217 217 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 119 119 119 218 218 218 F F F 15 300-319 109 109 109 205 205 205 F F F 16 320-339 118 118 118 217 217 217 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 119 119 119 218 218 218 F F F 19 380-399 119 119 119 218 218 218 F F F 20 400-419 119 119 119 218 218 218 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 119 119 119 218 218 218 F F F 23 460-479 116 116 116 214 214 214 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 116 116 116 214 214 214 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 119 119 119 218 218 218 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 119, Max flush: 218 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.189502153Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:18.242850541Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.243215207Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.281035035Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.281735448Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.281783115Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.281800761Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.281818698Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"Crucible 12bd9701-6edb-448d-904f-dc6cc62f9178 has session id: 11da08cc-cb51-48db-a46f-8eaf0dc24e97","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.28290009Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Upstairs opts: Upstairs UUID: 12bd9701-6edb-448d-904f-dc6cc62f9178, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.282950393Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Crucible stats registered with UUID: 12bd9701-6edb-448d-904f-dc6cc62f9178","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.282974433Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:18.283016719Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.283782816Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.283816505Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"12bd9701-6edb-448d-904f-dc6cc62f9178 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.283959578Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.283983467Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284170941Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284621162Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284655271Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284673659Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284691195Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284867716Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284898229Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284915504Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:36932)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284932629Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:47817)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284949754Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:47933)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.284970927Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.285209325Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.285423032Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.285884196Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.286131392Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.286886506Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.28695682Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.28697635Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.287973561Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[0]R flush_numbers[0..12]: [218, 213, 214, 214, 214, 212, 216, 217, 216, 215, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288256159Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[0]R generation[0..12]: [119, 114, 116, 116, 116, 113, 117, 118, 117, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.28828127Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288299107Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[1]R flush_numbers[0..12]: [218, 213, 214, 214, 214, 212, 216, 217, 216, 215, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288316592Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[1]R generation[0..12]: [119, 114, 116, 116, 116, 113, 117, 118, 117, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288333647Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288355442Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[2]R flush_numbers[0..12]: [218, 213, 214, 214, 214, 212, 216, 217, 216, 215, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288372427Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[2]R generation[0..12]: [119, 114, 116, 116, 116, 113, 117, 118, 117, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288389011Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288405775Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288421146Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.28843757Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288455206Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288472401Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"":"downstairs","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288488715Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288503495Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"0","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288519778Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"1","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288535891Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"client":"2","":"downstairs","session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288551914Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288566824Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"12bd9701-6edb-448d-904f-dc6cc62f9178 is now active with session: 11da08cc-cb51-48db-a46f-8eaf0dc24e97","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288582436Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288601395Z","hostname":"bmat-EVT22200007-0000589c","pid":1667,"session_id":"11da08cc-cb51-48db-a46f-8eaf0dc24e97"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288616596Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} {"msg":"Activated sub_volume 12bd9701-6edb-448d-904f-dc6cc62f9178","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:18.288630254Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 16384 data: 6 8 9 9 02/30 Write block 57 len 28672 data: 13 11 11 14 12 9 7 03/30 Read block 73 len 36864 04/30 Flush 05/30 Write block 257 len 8192 data: 15 12 06/30 Read block 31 len 16384 07/30 Read block 50 len 12288 08/30 Read block 261 len 28672 09/30 Read block 447 len 8192 10/30 Write block 38 len 20480 data: 11 11 13 13 13 11/30 Write block 22 len 16384 data: 7 8 8 9 12/30 Read block 440 len 16384 13/30 Read block 16 len 8192 14/30 Read block 100 len 8192 15/30 Write block 503 len 36864 data: 10 10 9 11 11 11 10 12 11 16/30 Read block 72 len 32768 17/30 Read block 232 len 20480 18/30 Write block 179 len 20480 data: 10 9 8 5 5 19/30 Read block 462 len 4096 20/30 Read block 443 len 24576 21/30 Read block 458 len 12288 22/30 Write block 464 len 32768 data: 13 14 12 11 12 13 11 10 23/30 Flush 24/30 Write block 10 len 24576 data: 7 6 9 9 10 7 25/30 Write block 458 len 8192 data: 9 9 26/30 Read block 57 len 36864 27/30 Write block 318 len 36864 data: 10 11 13 11 10 7 8 8 6 28/30 Write block 282 len 36864 data: 10 10 7 6 6 7 6 9 9 29/30 Read block 222 len 12288 30/30 Read block 145 len 16384 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:18.317346937Z","hostname":"bmat-EVT22200007-0000589c","pid":1667} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 218 218 218 T T T 1 020-039 120 120 120 220 220 220 F F F 2 040-059 120 120 120 220 220 220 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 117 117 117 216 216 216 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 120 120 120 220 220 220 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 119 119 119 218 218 218 T T T 15 300-319 109 109 109 205 205 205 T T T 16 320-339 118 118 118 217 217 217 T T T 17 340-359 114 114 114 213 213 213 F F F 18 360-379 119 119 119 218 218 218 F F F 19 380-399 119 119 119 218 218 218 F F F 20 400-419 119 119 119 218 218 218 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 119 119 119 218 218 218 T T T 23 460-479 120 120 120 220 220 220 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 120 120 120 220 220 220 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 119 119 119 218 218 218 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 120, Max flush: 220 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 221 221 221 F F F 1 020-039 120 120 120 220 220 220 F F F 2 040-059 120 120 120 220 220 220 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 F F F 6 120-139 117 117 117 216 216 216 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 120 120 120 220 220 220 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 121 121 121 221 221 221 F F F 15 300-319 121 121 121 221 221 221 F F F 16 320-339 121 121 121 221 221 221 F F F 17 340-359 114 114 114 213 213 213 F F F 18 360-379 119 119 119 218 218 218 F F F 19 380-399 119 119 119 218 218 218 F F F 20 400-419 119 119 119 218 218 218 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 F F F 24 480-499 105 105 105 194 194 194 F F F 25 500-519 120 120 120 220 220 220 F F F 26 520-539 113 113 113 212 212 212 F F F 27 540-559 119 119 119 218 218 218 F F F 28 560-579 114 114 114 213 213 213 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 121, Max flush: 221 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.682812337Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:22.733909118Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.734404652Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.770753199Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.771432129Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.77151582Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.771546012Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.771576754Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"Crucible c0af877e-d2e4-4585-b4a5-1655a3373f4a has session id: 14f172be-acca-4f35-b8f2-f1f4d226213e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.772488269Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Upstairs opts: Upstairs UUID: c0af877e-d2e4-4585-b4a5-1655a3373f4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.772537039Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Crucible stats registered with UUID: c0af877e-d2e4-4585-b4a5-1655a3373f4a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.772555838Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:22.77257143Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773345503Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773378801Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"c0af877e-d2e4-4585-b4a5-1655a3373f4a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773497234Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773520451Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773552817Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773569682Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773587198Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.77360776Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773625536Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773641359Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773657902Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773686591Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57788)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.773981324Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41336)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.774118966Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65201)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.774139317Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.774938933Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.774981591Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.775000499Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.775699289Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.775961926Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.775987939Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776457911Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776826334Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[0]R flush_numbers[0..12]: [221, 220, 220, 219, 214, 212, 216, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776937552Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[0]R generation[0..12]: [121, 120, 120, 120, 116, 113, 117, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776963966Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776982093Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[1]R flush_numbers[0..12]: [221, 220, 220, 219, 214, 212, 216, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.776999819Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[1]R generation[0..12]: [121, 120, 120, 120, 116, 113, 117, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777017405Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777034751Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[2]R flush_numbers[0..12]: [221, 220, 220, 219, 214, 212, 216, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777051134Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[2]R generation[0..12]: [121, 120, 120, 120, 116, 113, 117, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777067768Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777085104Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777100525Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777116308Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777131589Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777153995Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"":"downstairs","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777169867Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777184217Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"0","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777200891Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"1","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777216853Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"client":"2","":"downstairs","session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777232415Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777247145Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"c0af877e-d2e4-4585-b4a5-1655a3373f4a is now active with session: 14f172be-acca-4f35-b8f2-f1f4d226213e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777262567Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777276976Z","hostname":"bmat-EVT22200007-0000589c","pid":1675,"session_id":"14f172be-acca-4f35-b8f2-f1f4d226213e"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777291747Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} {"msg":"Activated sub_volume c0af877e-d2e4-4585-b4a5-1655a3373f4a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:22.777305465Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 16384 data: 13 13 13 13 02/30 Write block 504 len 8192 data: 11 10 03/30 Write block 1 len 40960 data: 3 6 8 8 12 14 14 11 10 8 04/30 Write block 525 len 40960 data: 7 9 10 11 9 9 9 7 6 7 05/30 Flush 06/30 Write block 360 len 36864 data: 9 8 6 9 11 12 12 8 8 07/30 Read block 66 len 12288 08/30 Read block 476 len 28672 09/30 Write block 358 len 20480 data: 11 13 10 9 7 10/30 Write block 405 len 20480 data: 13 11 11 14 14 11/30 Write block 296 len 12288 data: 13 13 11 12/30 Write block 293 len 32768 data: 10 12 13 14 14 12 9 9 13/30 Write block 120 len 16384 data: 8 6 6 4 14/30 Flush 15/30 Write block 353 len 32768 data: 6 6 8 10 11 12 14 11 16/30 Read block 164 len 32768 17/30 Read block 106 len 16384 18/30 Write block 563 len 12288 data: 6 7 9 19/30 Write block 123 len 16384 data: 5 8 8 9 20/30 Write block 125 len 24576 data: 9 10 9 9 8 9 21/30 Write block 492 len 24576 data: 5 7 5 4 4 5 22/30 Read block 15 len 4096 23/30 Read block 187 len 40960 24/30 Flush 25/30 Write block 49 len 16384 data: 9 10 9 14 26/30 Write block 115 len 24576 data: 14 15 12 11 8 9 27/30 Write block 489 len 36864 data: 9 7 6 6 8 6 5 5 6 28/30 Read block 592 len 4096 29/30 Write block 400 len 8192 data: 11 13 30/30 Read block 372 len 40960 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:22.80255296Z","hostname":"bmat-EVT22200007-0000589c","pid":1675} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 120 120 120 220 220 220 F F F 2 040-059 120 120 120 220 220 220 T T T 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 113 113 113 212 212 212 T T T 6 120-139 122 122 122 224 224 224 T T T 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 120 120 120 220 220 220 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 122 122 122 223 223 223 F F F 15 300-319 122 122 122 223 223 223 F F F 16 320-339 121 121 121 221 221 221 F F F 17 340-359 122 122 122 224 224 224 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 119 119 119 218 218 218 F F F 20 400-419 122 122 122 223 223 223 T T T 21 420-439 116 116 116 215 215 215 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 F F F 24 480-499 122 122 122 224 224 224 T T T 25 500-519 122 122 122 222 222 222 F F F 26 520-539 122 122 122 222 222 222 F F F 27 540-559 119 119 119 218 218 218 F F F 28 560-579 122 122 122 224 224 224 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 122, Max flush: 224 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 120 120 120 220 220 220 F F F 2 040-059 123 123 123 225 225 225 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 123 123 123 225 225 225 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 120 120 120 220 220 220 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 118 118 118 217 217 217 F F F 12 240-259 120 120 120 220 220 220 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 122 122 122 223 223 223 F F F 15 300-319 122 122 122 223 223 223 F F F 16 320-339 121 121 121 221 221 221 F F F 17 340-359 122 122 122 224 224 224 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 119 119 119 218 218 218 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 215 215 215 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 F F F 24 480-499 123 123 123 225 225 225 F F F 25 500-519 122 122 122 222 222 222 F F F 26 520-539 122 122 122 222 222 222 F F F 27 540-559 119 119 119 218 218 218 F F F 28 560-579 122 122 122 224 224 224 F F F 29 580-599 118 118 118 217 217 217 F F F Max gen: 123, Max flush: 225 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.167529951Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.221010229Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.221758961Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.221813202Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.221833052Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.221852272Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"Crucible 63ddee02-f7d1-4bce-9a8a-83e6f2e70510 has session id: 20526809-2ff1-4dd8-9121-8213a2d146be","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.222801113Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Upstairs opts: Upstairs UUID: 63ddee02-f7d1-4bce-9a8a-83e6f2e70510, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.222929315Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Crucible stats registered with UUID: 63ddee02-f7d1-4bce-9a8a-83e6f2e70510","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.222951851Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:27.222968425Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.223781158Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.22381137Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"63ddee02-f7d1-4bce-9a8a-83e6f2e70510 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.223990897Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224018965Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224115422Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224138469Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224157117Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224208973Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224230638Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224247472Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224265339Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"ds_connection connected from Ok(127.0.0.1:65097)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224283015Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224835555Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"ds_connection connected from Ok(127.0.0.1:54472)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224895869Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"ds_connection connected from Ok(127.0.0.1:58665)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.224915248Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.225081448Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.225277019Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.225560749Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.225851895Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.2258936Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.226212813Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.226681642Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.226759963Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[0]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 225, 225, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.226997649Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[0]R generation[0..12]: [122, 120, 123, 120, 116, 123, 123, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227023492Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.22704208Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[1]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 225, 225, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227060227Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[1]R generation[0..12]: [122, 120, 123, 120, 116, 123, 123, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227077893Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227096431Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[2]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 225, 225, 217, 220, 220, 209, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227113947Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[2]R generation[0..12]: [122, 120, 123, 120, 116, 123, 123, 118, 120, 120, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227138267Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227156514Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227172757Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227189431Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227205564Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227221787Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"":"downstairs","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227238552Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227253462Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227285558Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"1","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227302823Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"client":"2","":"downstairs","session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227319688Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227334949Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"63ddee02-f7d1-4bce-9a8a-83e6f2e70510 is now active with session: 20526809-2ff1-4dd8-9121-8213a2d146be","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227351202Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227366163Z","hostname":"bmat-EVT22200007-0000589c","pid":1683,"session_id":"20526809-2ff1-4dd8-9121-8213a2d146be"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227381364Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} {"msg":"Activated sub_volume 63ddee02-f7d1-4bce-9a8a-83e6f2e70510","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:27.227395783Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 24576 data: 12 11 13 18 16 15 02/30 Write block 581 len 8192 data: 9 9 03/30 Flush 04/30 Write block 552 len 36864 data: 10 10 12 10 11 11 10 9 7 05/30 Flush 06/30 Write block 508 len 40960 data: 12 11 13 12 10 11 11 11 10 10 07/30 Read block 588 len 20480 08/30 Write block 291 len 28672 data: 10 9 11 13 14 15 15 09/30 Write block 534 len 24576 data: 8 8 10 9 10 10 10/30 Read block 537 len 8192 11/30 Write block 558 len 20480 data: 11 10 8 10 10 12/30 Write block 201 len 28672 data: 9 10 10 11 11 11 8 13/30 Read block 87 len 32768 14/30 Read block 93 len 8192 15/30 Read block 371 len 4096 16/30 Read block 95 len 8192 17/30 Read block 249 len 12288 18/30 Read block 158 len 24576 19/30 Write block 584 len 12288 data: 11 10 11 20/30 Read block 38 len 24576 21/30 Read block 463 len 16384 22/30 Read block 120 len 8192 23/30 Write block 182 len 40960 data: 6 6 7 9 10 10 8 7 10 11 24/30 Read block 465 len 8192 25/30 Write block 236 len 28672 data: 11 11 11 7 7 9 10 26/30 Write block 196 len 4096 data: 12 27/30 Read block 155 len 20480 28/30 Read block 194 len 16384 29/30 Write block 117 len 16384 data: 13 12 9 10 30/30 Write block 424 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 10 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:27.254162407Z","hostname":"bmat-EVT22200007-0000589c","pid":1683} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 120 120 120 220 220 220 F F F 2 040-059 123 123 123 225 225 225 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 123 123 123 225 225 225 T T T 6 120-139 123 123 123 225 225 225 T T T 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 120 120 120 220 220 220 T T T 10 200-219 112 112 112 209 209 209 T T T 11 220-239 118 118 118 217 217 217 T T T 12 240-259 120 120 120 220 220 220 T T T 13 260-279 119 119 119 218 218 218 F F F 14 280-299 122 122 122 223 223 223 T T T 15 300-319 122 122 122 223 223 223 F F F 16 320-339 121 121 121 221 221 221 F F F 17 340-359 122 122 122 224 224 224 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 124 124 124 226 226 226 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 215 215 215 T T T 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 F F F 24 480-499 123 123 123 225 225 225 F F F 25 500-519 122 122 122 222 222 222 T T T 26 520-539 122 122 122 222 222 222 T T T 27 540-559 124 124 124 227 227 227 T T T 28 560-579 124 124 124 227 227 227 T T T 29 580-599 124 124 124 226 226 226 T T T Max gen: 124, Max flush: 227 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 120 120 120 220 220 220 F F F 2 040-059 123 123 123 225 225 225 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 116 116 116 214 214 214 F F F 5 100-119 125 125 125 228 228 228 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 118 118 118 217 217 217 F F F 8 160-179 120 120 120 220 220 220 F F F 9 180-199 125 125 125 228 228 228 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 119 119 119 218 218 218 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 122 122 122 223 223 223 F F F 16 320-339 121 121 121 221 221 221 F F F 17 340-359 122 122 122 224 224 224 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 124 124 124 226 226 226 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 125 125 125 228 228 228 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 F F F 24 480-499 123 123 123 225 225 225 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 125 125 125 228 228 228 F F F 28 560-579 125 125 125 228 228 228 F F F 29 580-599 125 125 125 228 228 228 F F F Max gen: 125, Max flush: 228 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.646897349Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.701639669Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.702394964Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.702449025Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.702467573Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.702486692Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"Crucible 9a819cc3-7c8d-48e4-a0f4-c33435c3176d has session id: 9cb0243a-bcc4-4b32-a32f-b12f1a1907ed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.70333608Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Upstairs opts: Upstairs UUID: 9a819cc3-7c8d-48e4-a0f4-c33435c3176d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.70338981Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Crucible stats registered with UUID: 9a819cc3-7c8d-48e4-a0f4-c33435c3176d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.703409209Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:31.703425312Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704179976Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.70441136Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"9a819cc3-7c8d-48e4-a0f4-c33435c3176d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704431942Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704447514Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704474399Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704491113Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704508048Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704531265Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704548891Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704564263Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704580827Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704598132Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:36940)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704676172Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:34844)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704696904Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:53429)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.704958028Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.706203717Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.706248529Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.706279793Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.706982079Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.707097556Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.707330091Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.707728016Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.707763077Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[0]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 228, 228, 217, 220, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708017317Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[0]R generation[0..12]: [122, 120, 123, 120, 116, 125, 125, 118, 120, 125, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708056548Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708076849Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[1]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 228, 228, 217, 220, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708094064Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[1]R generation[0..12]: [122, 120, 123, 120, 116, 125, 125, 118, 120, 125, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708111089Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708128224Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[2]R flush_numbers[0..12]: [222, 220, 225, 219, 214, 228, 228, 217, 220, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708145279Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[2]R generation[0..12]: [122, 120, 123, 120, 116, 125, 125, 118, 120, 125, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708162254Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708179509Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708195302Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708211244Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708227087Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708248581Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"":"downstairs","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708265345Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708279915Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"0","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708296138Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"1","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708312271Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"client":"2","":"downstairs","session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708328324Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708343074Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"9a819cc3-7c8d-48e4-a0f4-c33435c3176d is now active with session: 9cb0243a-bcc4-4b32-a32f-b12f1a1907ed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708358786Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708373667Z","hostname":"bmat-EVT22200007-0000589c","pid":1691,"session_id":"9cb0243a-bcc4-4b32-a32f-b12f1a1907ed"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708416454Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} {"msg":"Activated sub_volume 9a819cc3-7c8d-48e4-a0f4-c33435c3176d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:31.708431605Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 586 len 16384 data: 12 10 9 10 02/30 Write block 389 len 24576 data: 12 14 19 17 16 15 03/30 Write block 156 len 8192 data: 7 7 04/30 Read block 325 len 36864 05/30 Read block 257 len 4096 06/30 Write block 408 len 28672 data: 15 15 14 14 13 11 9 07/30 Read block 265 len 16384 08/30 Write block 540 len 4096 data: 9 09/30 Write block 279 len 4096 data: 12 10/30 Read block 63 len 36864 11/30 Write block 306 len 4096 data: 7 12/30 Write block 22 len 32768 data: 8 9 9 10 10 10 9 8 13/30 Write block 382 len 16384 data: 10 6 6 8 14/30 Write block 325 len 8192 data: 9 7 15/30 Read block 112 len 20480 16/30 Read block 413 len 36864 17/30 Write block 342 len 4096 data: 17 18/30 Flush 19/30 Write block 175 len 8192 data: 11 9 20/30 Read block 36 len 16384 21/30 Write block 81 len 24576 data: 12 11 10 10 10 9 22/30 Write block 311 len 4096 data: 7 23/30 Write block 189 len 16384 data: 8 11 12 12 24/30 Write block 197 len 12288 data: 12 11 8 25/30 Flush 26/30 Write block 432 len 12288 data: 13 14 11 27/30 Write block 476 len 40960 data: 7 8 9 9 10 10 11 12 8 10 28/30 Read block 352 len 8192 29/30 Read block 375 len 32768 30/30 Read block 409 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:31.731857814Z","hostname":"bmat-EVT22200007-0000589c","pid":1691} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 126 126 126 229 229 229 F F F 2 040-059 123 123 123 225 225 225 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 126 126 126 230 230 230 F F F 5 100-119 125 125 125 228 228 228 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 126 126 126 229 229 229 F F F 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 230 230 230 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 126 126 126 230 230 230 F F F 16 320-339 126 126 126 229 229 229 F F F 17 340-359 126 126 126 229 229 229 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 126 126 126 229 229 229 F F F 20 400-419 126 126 126 229 229 229 F F F 21 420-439 125 125 125 228 228 228 T T T 22 440-459 121 121 121 221 221 221 F F F 23 460-479 120 120 120 220 220 220 T T T 24 480-499 123 123 123 225 225 225 T T T 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 126 126 126 229 229 229 F F F 28 560-579 125 125 125 228 228 228 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 126, Max flush: 230 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 126 126 126 229 229 229 F F F 2 040-059 123 123 123 225 225 225 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 126 126 126 230 230 230 F F F 5 100-119 125 125 125 228 228 228 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 126 126 126 229 229 229 F F F 8 160-179 126 126 126 230 230 230 F F F 9 180-199 126 126 126 230 230 230 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 126 126 126 230 230 230 F F F 16 320-339 126 126 126 229 229 229 F F F 17 340-359 126 126 126 229 229 229 F F F 18 360-379 122 122 122 224 224 224 F F F 19 380-399 126 126 126 229 229 229 F F F 20 400-419 126 126 126 229 229 229 F F F 21 420-439 127 127 127 231 231 231 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 126 126 126 229 229 229 F F F 28 560-579 125 125 125 228 228 228 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 127, Max flush: 231 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.041503369Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.097127393Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.097940316Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.09799585Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.09801542Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.098034108Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"Crucible 248ceebb-916c-490b-90be-4ca1c73021a1 has session id: cdaed804-b9ba-44a9-9011-9d39e2992821","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.098828844Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Upstairs opts: Upstairs UUID: 248ceebb-916c-490b-90be-4ca1c73021a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.0989197Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Crucible stats registered with UUID: 248ceebb-916c-490b-90be-4ca1c73021a1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.098954241Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:36.098981857Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.099966071Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100188025Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"248ceebb-916c-490b-90be-4ca1c73021a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100207214Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100223447Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100253148Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100270393Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100288831Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100305255Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100368574Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100385328Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100402914Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100419929Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"ds_connection connected from Ok(127.0.0.1:53284)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100518872Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"ds_connection connected from Ok(127.0.0.1:39979)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100544043Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"ds_connection connected from Ok(127.0.0.1:34441)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.100561769Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.101608461Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.101765562Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.101794291Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.102405692Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.102506007Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.10272718Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103238797Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103374675Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[0]R flush_numbers[0..12]: [222, 229, 225, 219, 230, 228, 228, 229, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103498057Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[0]R generation[0..12]: [122, 126, 123, 120, 126, 125, 125, 126, 126, 126, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.10352379Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103542919Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[1]R flush_numbers[0..12]: [222, 229, 225, 219, 230, 228, 228, 229, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103561056Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[1]R generation[0..12]: [122, 126, 123, 120, 126, 125, 125, 126, 126, 126, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103578512Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.10359706Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[2]R flush_numbers[0..12]: [222, 229, 225, 219, 230, 228, 228, 229, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103614916Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[2]R generation[0..12]: [122, 126, 123, 120, 126, 125, 125, 126, 126, 126, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103640088Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103658345Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103675189Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103692134Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103708387Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103724621Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"":"downstairs","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103741535Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103756846Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"0","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103773541Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"1","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103790585Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"client":"2","":"downstairs","session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Wait for a query_work_queue command to finish before sending IO All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103819414Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103836059Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"248ceebb-916c-490b-90be-4ca1c73021a1 is now active with session: cdaed804-b9ba-44a9-9011-9d39e2992821","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103852171Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103867262Z","hostname":"bmat-EVT22200007-0000589c","pid":1699,"session_id":"cdaed804-b9ba-44a9-9011-9d39e2992821"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103882654Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} {"msg":"Activated sub_volume 248ceebb-916c-490b-90be-4ca1c73021a1","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:36.103897123Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 20480 data: 12 12 12 11 9 02/30 Read block 468 len 36864 03/30 Write block 53 len 24576 data: 16 15 13 11 14 12 04/30 Write block 370 len 4096 data: 7 05/30 Read block 102 len 12288 06/30 Read block 381 len 40960 07/30 Read block 84 len 40960 08/30 Read block 212 len 32768 09/30 Read block 504 len 16384 10/30 Write block 382 len 28672 data: 11 7 7 9 10 13 13 11/30 Write block 184 len 36864 data: 8 10 11 11 9 9 12 13 13 12/30 Read block 587 len 8192 13/30 Read block 215 len 36864 14/30 Write block 186 len 24576 data: 12 12 10 10 13 14 15/30 Read block 358 len 12288 16/30 Flush 17/30 Write block 143 len 36864 data: 14 16 15 13 10 10 9 7 7 18/30 Flush 19/30 Write block 28 len 16384 data: 10 9 9 9 20/30 Write block 404 len 4096 data: 13 21/30 Read block 550 len 28672 22/30 Write block 179 len 16384 data: 11 10 9 7 23/30 Write block 483 len 8192 data: 13 9 24/30 Write block 482 len 4096 data: 12 25/30 Write block 194 len 24576 data: 12 13 13 13 12 9 26/30 Flush 27/30 Write block 114 len 4096 data: 14 28/30 Read block 578 len 24576 29/30 Write block 182 len 20480 data: 8 7 9 11 13 30/30 Read block 149 len 12288 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:36.131670166Z","hostname":"bmat-EVT22200007-0000589c","pid":1699} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 128 128 128 232 232 232 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 126 126 126 230 230 230 F F F 5 100-119 125 125 125 228 228 228 T T T 6 120-139 125 125 125 228 228 228 F F F 7 140-159 128 128 128 233 233 233 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 128 128 128 234 234 234 T T T 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 126 126 126 230 230 230 F F F 16 320-339 126 126 126 229 229 229 F F F 17 340-359 126 126 126 229 229 229 F F F 18 360-379 128 128 128 232 232 232 F F F 19 380-399 128 128 128 232 232 232 F F F 20 400-419 128 128 128 234 234 234 F F F 21 420-439 127 127 127 231 231 231 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 128 128 128 232 232 232 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 128, Max flush: 234 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 128 128 128 234 234 234 F F F 2 040-059 128 128 128 232 232 232 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 126 126 126 230 230 230 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 128 128 128 233 233 233 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 129 129 129 235 235 235 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 126 126 126 230 230 230 F F F 16 320-339 126 126 126 229 229 229 F F F 17 340-359 126 126 126 229 229 229 F F F 18 360-379 128 128 128 232 232 232 F F F 19 380-399 128 128 128 232 232 232 F F F 20 400-419 128 128 128 234 234 234 F F F 21 420-439 127 127 127 231 231 231 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 128 128 128 232 232 232 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 129, Max flush: 235 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.43520997Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.490322497Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.491057351Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.491114287Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.491135491Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.491155442Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"Crucible b2569b8f-35bb-416a-b510-a3f31d40eb4a has session id: b7c3129c-50e7-41c6-b985-3c3c8e3f53cc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.492110846Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Upstairs opts: Upstairs UUID: b2569b8f-35bb-416a-b510-a3f31d40eb4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.492167422Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Crucible stats registered with UUID: b2569b8f-35bb-416a-b510-a3f31d40eb4a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.492188345Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:40.492205871Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.492937858Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.492965244Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"b2569b8f-35bb-416a-b510-a3f31d40eb4a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493096132Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493352447Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493850646Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493882621Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.49390174Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493918655Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493936632Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493952705Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493969679Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.493986033Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63965)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.494008268Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60454)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.494025454Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41050)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.494042168Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.495013124Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.49505019Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.495552828Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.496212699Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.496278393Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.496415994Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.496881677Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.496910867Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[0]R flush_numbers[0..12]: [222, 234, 232, 219, 230, 235, 228, 233, 234, 235, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497195018Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[0]R generation[0..12]: [122, 128, 128, 120, 126, 129, 125, 128, 128, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497218767Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497237455Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[1]R flush_numbers[0..12]: [222, 234, 232, 219, 230, 235, 228, 233, 234, 235, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497262366Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[1]R generation[0..12]: [122, 128, 128, 120, 126, 129, 125, 128, 128, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497279892Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497299332Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[2]R flush_numbers[0..12]: [222, 234, 232, 219, 230, 235, 228, 233, 234, 235, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497316928Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[2]R generation[0..12]: [122, 128, 128, 120, 126, 129, 125, 128, 128, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497334023Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497351338Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497367561Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497383764Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497399547Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497415129Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"":"downstairs","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497431372Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497446112Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"0","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497462506Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"1","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497478528Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"client":"2","":"downstairs","session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497494882Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.49751371Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"b2569b8f-35bb-416a-b510-a3f31d40eb4a is now active with session: b7c3129c-50e7-41c6-b985-3c3c8e3f53cc","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497529783Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497544594Z","hostname":"bmat-EVT22200007-0000589c","pid":1707,"session_id":"b7c3129c-50e7-41c6-b985-3c3c8e3f53cc"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497560085Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} {"msg":"Activated sub_volume b2569b8f-35bb-416a-b510-a3f31d40eb4a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:40.497574094Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 419 len 32768 data: 7 9 7 7 7 7 6 4 02/30 Flush 03/30 Write block 433 len 12288 data: 15 12 12 04/30 Flush 05/30 Write block 328 len 24576 data: 8 9 12 13 13 14 06/30 Read block 529 len 24576 07/30 Read block 131 len 28672 08/30 Read block 396 len 20480 09/30 Write block 34 len 24576 data: 13 12 9 10 12 12 10/30 Read block 56 len 28672 11/30 Read block 29 len 12288 12/30 Read block 354 len 4096 13/30 Read block 54 len 24576 14/30 Write block 309 len 12288 data: 5 7 8 15/30 Flush 16/30 Write block 95 len 32768 data: 11 12 13 12 11 10 7 9 17/30 Write block 181 len 8192 data: 10 9 18/30 Read block 98 len 32768 19/30 Write block 344 len 24576 data: 13 12 11 12 11 8 20/30 Write block 187 len 24576 data: 13 11 11 14 15 14 21/30 Read block 338 len 36864 22/30 Flush 23/30 Write block 46 len 36864 data: 11 11 8 10 11 10 15 17 16 24/30 Write block 568 len 28672 data: 12 10 11 9 8 7 8 25/30 Read block 145 len 36864 26/30 Read block 367 len 32768 27/30 Write block 327 len 16384 data: 8 9 10 13 28/30 Read block 302 len 24576 29/30 Read block 281 len 28672 30/30 Write block 380 len 36864 data: 14 14 12 8 8 10 11 14 14 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:40.525392729Z","hostname":"bmat-EVT22200007-0000589c","pid":1707} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 130 130 130 238 238 238 F F F 2 040-059 128 128 128 232 232 232 T T T 3 060-079 120 120 120 219 219 219 F F F 4 080-099 130 130 130 239 239 239 F F F 5 100-119 130 130 130 239 239 239 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 128 128 128 233 233 233 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 130 130 130 238 238 238 F F F 16 320-339 130 130 130 238 238 238 T T T 17 340-359 130 130 130 239 239 239 F F F 18 360-379 128 128 128 232 232 232 F F F 19 380-399 128 128 128 232 232 232 T T T 20 400-419 130 130 130 236 236 236 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 128 128 128 232 232 232 T T T 29 580-599 126 126 126 229 229 229 F F F Max gen: 130, Max flush: 239 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 130 130 130 238 238 238 F F F 2 040-059 131 131 131 240 240 240 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 130 130 130 239 239 239 F F F 5 100-119 130 130 130 239 239 239 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 128 128 128 233 233 233 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 F F F 13 260-279 126 126 126 229 229 229 F F F 14 280-299 125 125 125 228 228 228 F F F 15 300-319 130 130 130 238 238 238 F F F 16 320-339 131 131 131 240 240 240 F F F 17 340-359 130 130 130 239 239 239 F F F 18 360-379 128 128 128 232 232 232 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 130 130 130 236 236 236 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 121 121 121 221 221 221 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 125 125 125 228 228 228 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 131, Max flush: 240 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.840650987Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.899292936Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.900201455Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.900258822Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.900279114Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.900297421Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"Crucible 26e316a1-cb89-4988-8a6c-b43536d1b665 has session id: fdcdfe32-bb5a-4167-a709-bbc5f6dccf81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.901255361Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Upstairs opts: Upstairs UUID: 26e316a1-cb89-4988-8a6c-b43536d1b665, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.901384876Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Crucible stats registered with UUID: 26e316a1-cb89-4988-8a6c-b43536d1b665","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.901408665Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:44.901424417Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902545881Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902606064Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"26e316a1-cb89-4988-8a6c-b43536d1b665 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902718224Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902740279Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902770391Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902787446Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902810423Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902827217Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902845043Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902861948Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902934276Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.902956922Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"ds_connection connected from Ok(127.0.0.1:36096)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.903388756Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"ds_connection connected from Ok(127.0.0.1:55025)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.903427565Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"ds_connection connected from Ok(127.0.0.1:37485)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.903447967Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.904427691Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.904464066Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.904482864Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.905599329Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.905637216Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.905658059Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906291735Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906437814Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[0]R flush_numbers[0..12]: [222, 238, 240, 219, 239, 239, 228, 233, 234, 239, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906671352Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[0]R generation[0..12]: [122, 130, 131, 120, 130, 130, 125, 128, 128, 130, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906760835Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906794454Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[1]R flush_numbers[0..12]: [222, 238, 240, 219, 239, 239, 228, 233, 234, 239, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906826349Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} Wait for a query_work_queue command to finish before sending IO {"msg":"[1]R generation[0..12]: [122, 130, 131, 120, 130, 130, 125, 128, 128, 130, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906866611Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906897444Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[2]R flush_numbers[0..12]: [222, 238, 240, 219, 239, 239, 228, 233, 234, 239, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906926183Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[2]R generation[0..12]: [122, 130, 131, 120, 130, 130, 125, 128, 128, 130, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906954191Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.906982639Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907009123Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907036699Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907063304Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907097654Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"":"downstairs","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907126974Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907152306Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"0","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907181235Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"1","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907209393Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"client":"2","":"downstairs","session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907225786Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907240827Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"26e316a1-cb89-4988-8a6c-b43536d1b665 is now active with session: fdcdfe32-bb5a-4167-a709-bbc5f6dccf81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907256579Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907271119Z","hostname":"bmat-EVT22200007-0000589c","pid":1715,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 fdcdfe32-bb5a-4167-a709-bbc5f6dccf81"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907294717Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} {"msg":"Activated sub_volume 26e316a1-cb89-4988-8a6c-b43536d1b665","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:44.907308996Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 73 len 20480 data: 7 7 9 8 7 02/30 Write block 365 len 8192 data: 13 13 03/30 Read block 409 len 8192 04/30 Write block 94 len 36864 data: 10 12 13 14 13 12 11 8 10 05/30 Write block 274 len 16384 data: 9 9 9 9 06/30 Write block 195 len 4096 data: 14 07/30 Flush 08/30 Write block 452 len 16384 data: 9 9 7 7 09/30 Read block 93 len 32768 10/30 Write block 74 len 4096 data: 8 11/30 Write block 150 len 8192 data: 8 8 12/30 Read block 510 len 40960 13/30 Flush 14/30 Write block 109 len 40960 data: 11 13 12 13 16 15 15 16 14 13 15/30 Write block 315 len 32768 data: 8 9 9 11 12 14 12 11 16/30 Read block 39 len 4096 17/30 Write block 94 len 40960 data: 11 13 14 15 14 13 12 9 11 8 18/30 Write block 405 len 8192 data: 14 12 19/30 Write block 248 len 36864 data: 10 11 11 11 9 10 9 13 14 20/30 Write block 114 len 12288 data: 16 16 17 21/30 Read block 415 len 16384 22/30 Read block 584 len 16384 23/30 Read block 323 len 4096 24/30 Write block 294 len 16384 data: 14 15 16 16 25/30 Read block 442 len 40960 26/30 Write block 506 len 28672 data: 12 12 13 12 14 13 11 27/30 Read block 45 len 12288 28/30 Read block 505 len 32768 29/30 Write block 55 len 24576 data: 14 12 15 13 12 15 30/30 Read block 499 len 20480 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 17 15 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:44.931332097Z","hostname":"bmat-EVT22200007-0000589c","pid":1715} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 130 130 130 238 238 238 F F F 2 040-059 131 131 131 240 240 240 T T T 3 060-079 132 132 132 242 242 242 T T T 4 080-099 132 132 132 241 241 241 T T T 5 100-119 132 132 132 241 241 241 T T T 6 120-139 125 125 125 228 228 228 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 132 132 132 241 241 241 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 125 125 125 228 228 228 T T T 13 260-279 132 132 132 241 241 241 F F F 14 280-299 125 125 125 228 228 228 T T T 15 300-319 130 130 130 238 238 238 T T T 16 320-339 131 131 131 240 240 240 T T T 17 340-359 130 130 130 239 239 239 F F F 18 360-379 132 132 132 241 241 241 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 130 130 130 236 236 236 T T T 21 420-439 130 130 130 237 237 237 F F F 22 440-459 132 132 132 242 242 242 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 125 125 125 228 228 228 T T T 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 132, Max flush: 242 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 130 130 130 238 238 238 F F F 2 040-059 133 133 133 243 243 243 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 133 133 133 243 243 243 F F F 5 100-119 133 133 133 243 243 243 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 132 132 132 241 241 241 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 132 132 132 241 241 241 F F F 14 280-299 133 133 133 243 243 243 F F F 15 300-319 133 133 133 243 243 243 F F F 16 320-339 133 133 133 243 243 243 F F F 17 340-359 130 130 130 239 239 239 F F F 18 360-379 132 132 132 241 241 241 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 133 133 133 243 243 243 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 132 132 132 242 242 242 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 133 133 133 243 243 243 F F F 26 520-539 125 125 125 228 228 228 F F F 27 540-559 128 128 128 232 232 232 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 126 126 126 229 229 229 F F F Max gen: 133, Max flush: 243 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.325550114Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:49.377037274Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.377456462Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.41365398Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.415229985Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.415265377Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.415281741Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.415298906Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"Crucible 1516c3c0-64b4-40c4-82b8-22a569acef81 has session id: 65dbe712-e2c2-481e-b64b-891261d53367","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.416503631Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Upstairs opts: Upstairs UUID: 1516c3c0-64b4-40c4-82b8-22a569acef81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.416560216Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Crucible stats registered with UUID: 1516c3c0-64b4-40c4-82b8-22a569acef81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.416579686Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:49.416595148Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417488005Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417563509Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"1516c3c0-64b4-40c4-82b8-22a569acef81 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417581466Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417598671Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417635967Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417653383Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.41767156Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417687633Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417705219Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417721292Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.417757105Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"ds_connection connected from Ok(127.0.0.1:36863)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.418053031Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.418099526Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"ds_connection connected from Ok(127.0.0.1:37455)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.418119076Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"ds_connection connected from Ok(127.0.0.1:65173)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.418409921Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.419066093Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.419247474Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.419438525Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.419838003Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.420093535Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.420371264Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.420798718Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.420860785Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[0]R flush_numbers[0..12]: [222, 238, 243, 243, 243, 243, 228, 242, 234, 241, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421318291Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[0]R generation[0..12]: [122, 130, 133, 133, 133, 133, 125, 132, 128, 132, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421344174Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421362041Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[1]R flush_numbers[0..12]: [222, 238, 243, 243, 243, 243, 228, 242, 234, 241, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421385749Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[1]R generation[0..12]: [122, 130, 133, 133, 133, 133, 125, 132, 128, 132, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421403916Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421421693Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[2]R flush_numbers[0..12]: [222, 238, 243, 243, 243, 243, 228, 242, 234, 241, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421438657Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[2]R generation[0..12]: [122, 130, 133, 133, 133, 133, 125, 132, 128, 132, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421455111Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421471685Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421487397Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421503099Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.42151818Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421533672Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"":"downstairs","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421549454Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421564365Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421593725Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"1","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421610038Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"client":"2","":"downstairs","session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421626011Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421644048Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"1516c3c0-64b4-40c4-82b8-22a569acef81 is now active with session: 65dbe712-e2c2-481e-b64b-891261d53367","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.42165991Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421674239Z","hostname":"bmat-EVT22200007-0000589c","pid":1723,"session_id":"65dbe712-e2c2-481e-b64b-891261d53367"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.42168922Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} {"msg":"Activated sub_volume 1516c3c0-64b4-40c4-82b8-22a569acef81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:49.421702928Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 378 len 4096 data: 15 02/30 Write block 364 len 12288 data: 12 14 14 03/30 Read block 260 len 4096 04/30 Write block 102 len 40960 data: 12 9 10 11 11 10 12 12 14 13 05/30 Read block 237 len 20480 06/30 Write block 361 len 36864 data: 10 8 10 13 15 15 9 9 6 07/30 Flush 08/30 Write block 538 len 36864 data: 11 11 10 9 10 9 9 5 8 09/30 Read block 571 len 36864 10/30 Read block 220 len 36864 11/30 Read block 173 len 24576 12/30 Write block 586 len 28672 data: 13 11 10 11 8 10 8 13/30 Write block 348 len 4096 data: 12 14/30 Read block 107 len 40960 15/30 Read block 378 len 36864 16/30 Read block 596 len 4096 17/30 Read block 288 len 20480 18/30 Read block 538 len 40960 19/30 Flush 20/30 Flush 21/30 Write block 36 len 40960 data: 10 11 13 13 14 14 14 13 11 11 22/30 Read block 292 len 32768 23/30 Read block 289 len 24576 24/30 Write block 328 len 40960 data: 10 11 14 14 14 15 14 11 12 12 25/30 Read block 496 len 40960 26/30 Flush 27/30 Write block 376 len 8192 data: 17 14 28/30 Write block 96 len 8192 data: 15 16 29/30 Write block 195 len 8192 data: 15 14 30/30 Read block 552 len 16384 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:49.450857798Z","hostname":"bmat-EVT22200007-0000589c","pid":1723} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 134 134 134 247 247 247 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 133 133 133 243 243 243 T T T 5 100-119 134 134 134 244 244 244 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 132 132 132 241 241 241 T T T 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 132 132 132 241 241 241 F F F 14 280-299 133 133 133 243 243 243 F F F 15 300-319 133 133 133 243 243 243 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 134 134 134 245 245 245 F F F 18 360-379 134 134 134 244 244 244 T T T 19 380-399 131 131 131 240 240 240 F F F 20 400-419 133 133 133 243 243 243 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 132 132 132 242 242 242 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 133 133 133 243 243 243 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 134 134 134 245 245 245 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 134, Max flush: 247 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 222 222 222 F F F 1 020-039 134 134 134 247 247 247 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 134 134 134 244 244 244 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 132 132 132 242 242 242 F F F 8 160-179 128 128 128 234 234 234 F F F 9 180-199 135 135 135 248 248 248 F F F 10 200-219 125 125 125 228 228 228 F F F 11 220-239 125 125 125 228 228 228 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 132 132 132 241 241 241 F F F 14 280-299 133 133 133 243 243 243 F F F 15 300-319 133 133 133 243 243 243 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 134 134 134 245 245 245 F F F 18 360-379 135 135 135 248 248 248 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 133 133 133 243 243 243 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 132 132 132 242 242 242 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 133 133 133 243 243 243 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 134 134 134 245 245 245 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 135, Max flush: 248 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.79726281Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.852393735Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.853220126Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.853288626Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.853308367Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.853327315Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"Crucible e42eb005-fa6f-4ed6-bb21-d474613fe298 has session id: 54ed66fe-613e-4cde-890d-abaa6458bd91","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.854250935Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Upstairs opts: Upstairs UUID: e42eb005-fa6f-4ed6-bb21-d474613fe298, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.854301689Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Crucible stats registered with UUID: e42eb005-fa6f-4ed6-bb21-d474613fe298","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.854323333Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:53.854339236Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855167841Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85523556Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"e42eb005-fa6f-4ed6-bb21-d474613fe298 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855253086Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855271844Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855306665Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85532372Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855341096Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855364073Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855381769Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855398563Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855421781Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855438345Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:58707)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855921092Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:61980)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855965794Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"ds_connection connected from Ok(127.0.0.1:48368)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.855996006Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.856848099Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.857163424Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.857226734Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.857800908Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.857845179Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858165935Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858448263Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858565583Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[0]R flush_numbers[0..12]: [222, 247, 247, 243, 248, 244, 228, 242, 234, 248, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858931201Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[0]R generation[0..12]: [122, 134, 134, 133, 135, 134, 125, 132, 128, 135, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858954408Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858973177Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[1]R flush_numbers[0..12]: [222, 247, 247, 243, 248, 244, 228, 242, 234, 248, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.858990292Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[1]R generation[0..12]: [122, 134, 134, 133, 135, 134, 125, 132, 128, 135, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859006565Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85902371Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[2]R flush_numbers[0..12]: [222, 247, 247, 243, 248, 244, 228, 242, 234, 248, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859040224Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[2]R generation[0..12]: [122, 134, 134, 133, 135, 134, 125, 132, 128, 135, 125, 125]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859056527Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859073081Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859088272Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859103664Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859118704Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859134397Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"":"downstairs","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859154628Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859169338Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"0","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859196734Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"1","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859213629Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"client":"2","":"downstairs","session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85922897Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859242989Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"e42eb005-fa6f-4ed6-bb21-d474613fe298 is now active with session: 54ed66fe-613e-4cde-890d-abaa6458bd91","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85925831Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.85927289Z","hostname":"bmat-EVT22200007-0000589c","pid":1731,"session_id":"54ed66fe-613e-4cde-890d-abaa6458bd91"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859286969Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} {"msg":"Activated sub_volume e42eb005-fa6f-4ed6-bb21-d474613fe298","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:53.859300296Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 40960 data: 14 11 11 14 15 13 12 16 16 15 02/30 Read block 184 len 28672 03/30 Read block 483 len 24576 04/30 Write block 492 len 8192 data: 7 9 05/30 Write block 8 len 8192 data: 12 11 06/30 Write block 491 len 12288 data: 7 8 10 07/30 Write block 501 len 36864 data: 10 10 11 12 11 13 13 14 13 08/30 Read block 560 len 12288 09/30 Flush 10/30 Write block 357 len 28672 data: 12 13 15 12 11 9 11 11/30 Write block 439 len 8192 data: 8 8 12/30 Read block 131 len 32768 13/30 Read block 470 len 20480 14/30 Write block 150 len 12288 data: 9 9 5 15/30 Write block 439 len 40960 data: 9 9 8 9 8 10 10 12 11 13 16/30 Flush 17/30 Write block 248 len 36864 data: 11 12 12 12 10 11 10 14 15 18/30 Write block 350 len 32768 data: 8 7 9 7 7 9 11 13 19/30 Write block 165 len 12288 data: 11 10 16 20/30 Flush 21/30 Write block 231 len 20480 data: 9 11 13 9 10 22/30 Write block 294 len 40960 data: 15 16 17 17 13 10 10 8 8 7 23/30 Read block 109 len 12288 24/30 Write block 203 len 16384 data: 11 12 12 12 25/30 Write block 32 len 28672 data: 10 8 14 13 11 12 14 26/30 Read block 504 len 32768 27/30 Read block 530 len 28672 28/30 Read block 389 len 8192 29/30 Read block 576 len 8192 30/30 Write block 349 len 40960 data: 9 9 8 10 8 8 10 12 14 14 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:53.886807394Z","hostname":"bmat-EVT22200007-0000589c","pid":1731} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 134 134 134 247 247 247 T T T 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 134 134 134 244 244 244 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 136 136 136 251 251 251 F F F 9 180-199 135 135 135 248 248 248 F F F 10 200-219 125 125 125 228 228 228 T T T 11 220-239 125 125 125 228 228 228 T T T 12 240-259 136 136 136 251 251 251 F F F 13 260-279 132 132 132 241 241 241 F F F 14 280-299 133 133 133 243 243 243 T T T 15 300-319 133 133 133 243 243 243 T T T 16 320-339 134 134 134 247 247 247 F F F 17 340-359 136 136 136 251 251 251 T T T 18 360-379 136 136 136 250 250 250 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 136 136 136 249 249 249 F F F 21 420-439 136 136 136 250 250 250 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 136 136 136 249 249 249 F F F 25 500-519 136 136 136 249 249 249 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 134 134 134 245 245 245 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 136, Max flush: 251 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 137 137 137 252 252 252 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 134 134 134 244 244 244 F F F 6 120-139 125 125 125 228 228 228 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 136 136 136 251 251 251 F F F 9 180-199 135 135 135 248 248 248 F F F 10 200-219 137 137 137 252 252 252 F F F 11 220-239 137 137 137 252 252 252 F F F 12 240-259 136 136 136 251 251 251 F F F 13 260-279 132 132 132 241 241 241 F F F 14 280-299 137 137 137 252 252 252 F F F 15 300-319 137 137 137 252 252 252 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 131 131 131 240 240 240 F F F 20 400-419 136 136 136 249 249 249 F F F 21 420-439 136 136 136 250 250 250 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 136 136 136 249 249 249 F F F 25 500-519 136 136 136 249 249 249 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 134 134 134 245 245 245 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 137, Max flush: 252 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.221655232Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.275074035Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.275854902Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.275903031Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.2759221Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.275939796Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"Crucible 9b77762a-2e44-426d-bb12-04adace79e6e has session id: ecdb295a-739e-437a-85dd-a8c9b00deb6a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.27690478Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Upstairs opts: Upstairs UUID: 9b77762a-2e44-426d-bb12-04adace79e6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.276953771Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Crucible stats registered with UUID: 9b77762a-2e44-426d-bb12-04adace79e6e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.27697304Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:58.276988422Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277741943Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277771103Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"9b77762a-2e44-426d-bb12-04adace79e6e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.2778917Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277913053Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277943846Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277962605Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.27797992Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.277995722Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278013208Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278029602Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278205231Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278233429Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49342)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278251025Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55449)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278272639Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38928)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.278371221Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.279217532Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.279258276Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.280077912Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.280239513Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.280869672Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.2810249Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.281975565Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.282966132Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[0]R flush_numbers[0..12]: [249, 252, 247, 243, 248, 244, 228, 250, 251, 248, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.283923059Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[0]R generation[0..12]: [136, 137, 134, 133, 135, 134, 125, 136, 136, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.28399196Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284013214Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[1]R flush_numbers[0..12]: [249, 252, 247, 243, 248, 244, 228, 250, 251, 248, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284031952Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[1]R generation[0..12]: [136, 137, 134, 133, 135, 134, 125, 136, 136, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284049498Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284067224Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[2]R flush_numbers[0..12]: [249, 252, 247, 243, 248, 244, 228, 250, 251, 248, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284091745Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[2]R generation[0..12]: [136, 137, 134, 133, 135, 134, 125, 136, 136, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.28410927Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284127287Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284143661Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284159864Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284175316Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284190958Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"":"downstairs","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.28420677Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284221761Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"0","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284238475Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"1","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284255059Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"client":"2","":"downstairs","session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284271382Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284286072Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"9b77762a-2e44-426d-bb12-04adace79e6e is now active with session: ecdb295a-739e-437a-85dd-a8c9b00deb6a","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284301494Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284315723Z","hostname":"bmat-EVT22200007-0000589c","pid":1739,"session_id":"ecdb295a-739e-437a-85dd-a8c9b00deb6a"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:08:58.284330854Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} {"msg":"Activated sub_volume 9b77762a-2e44-426d-bb12-04adace79e6e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:08:58.284365425Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 552 len 16384 data: 11 11 13 11 03/30 Read block 174 len 20480 04/30 Write block 147 len 16384 data: 11 11 10 10 05/30 Read block 34 len 24576 06/30 Write block 428 len 20480 data: 7 9 10 11 14 07/30 Write block 546 len 12288 data: 9 10 11 08/30 Flush 09/30 Write block 141 len 32768 data: 12 10 15 17 16 14 12 12 10/30 Read block 148 len 24576 11/30 Write block 261 len 8192 data: 9 10 12/30 Write block 124 len 28672 data: 9 10 11 10 10 9 10 13/30 Read block 309 len 4096 14/30 Write block 297 len 36864 data: 18 14 11 11 9 9 8 7 6 15/30 Flush 16/30 Flush 17/30 Write block 231 len 40960 data: 10 12 14 10 11 12 12 12 8 8 18/30 Write block 383 len 32768 data: 9 9 11 12 15 15 13 15 19/30 Read block 9 len 40960 20/30 Read block 504 len 24576 21/30 Write block 26 len 4096 data: 11 22/30 Read block 467 len 8192 23/30 Read block 440 len 12288 24/30 Read block 121 len 32768 25/30 Read block 188 len 8192 26/30 Flush 27/30 Write block 33 len 24576 data: 9 15 14 12 13 15 28/30 Write block 429 len 28672 data: 10 11 12 15 16 13 13 29/30 Write block 30 len 40960 data: 10 10 11 10 16 15 13 14 16 14 30/30 Write block 220 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:08:58.311776997Z","hostname":"bmat-EVT22200007-0000589c","pid":1739} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 138 138 138 257 257 257 T T T 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 134 134 134 244 244 244 F F F 6 120-139 138 138 138 255 255 255 F F F 7 140-159 138 138 138 255 255 255 F F F 8 160-179 136 136 136 251 251 251 F F F 9 180-199 135 135 135 248 248 248 F F F 10 200-219 137 137 137 252 252 252 F F F 11 220-239 138 138 138 257 257 257 T T T 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 138 138 138 255 255 255 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 138 138 138 257 257 257 F F F 20 400-419 136 136 136 249 249 249 F F F 21 420-439 138 138 138 254 254 254 T T T 22 440-459 136 136 136 250 250 250 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 136 136 136 249 249 249 F F F 25 500-519 136 136 136 249 249 249 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 138 138 138 254 254 254 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 138, Max flush: 257 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 139 139 139 258 258 258 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 F F F 4 080-099 135 135 135 248 248 248 F F F 5 100-119 134 134 134 244 244 244 F F F 6 120-139 138 138 138 255 255 255 F F F 7 140-159 138 138 138 255 255 255 F F F 8 160-179 136 136 136 251 251 251 F F F 9 180-199 135 135 135 248 248 248 F F F 10 200-219 137 137 137 252 252 252 F F F 11 220-239 139 139 139 258 258 258 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 138 138 138 255 255 255 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 138 138 138 257 257 257 F F F 20 400-419 136 136 136 249 249 249 F F F 21 420-439 139 139 139 258 258 258 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 127 127 127 231 231 231 F F F 24 480-499 136 136 136 249 249 249 F F F 25 500-519 136 136 136 249 249 249 F F F 26 520-539 134 134 134 245 245 245 F F F 27 540-559 138 138 138 254 254 254 F F F 28 560-579 131 131 131 240 240 240 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 139, Max flush: 258 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.626895621Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.680384026Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.681268285Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.681318808Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.681336715Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.681354702Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"Crucible f4928bdb-72a5-44f9-a1ce-a09befbc8d3d has session id: 26c4e620-a31e-490e-9f85-77be580fbd89","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.682121821Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Upstairs opts: Upstairs UUID: f4928bdb-72a5-44f9-a1ce-a09befbc8d3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.682172024Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Crucible stats registered with UUID: f4928bdb-72a5-44f9-a1ce-a09befbc8d3d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.682190592Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:02.682206615Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.682834149Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683026312Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"f4928bdb-72a5-44f9-a1ce-a09befbc8d3d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683070382Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683087387Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683115574Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68313321Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683150746Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683166679Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683183974Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683200197Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683216621Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683241823Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:54129)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683295903Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:59664)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683319081Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:46707)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.683339733Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68429133Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.684474004Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.684944506Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.685223126Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.685693719Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68595793Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68704818Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.687805249Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[0]R flush_numbers[0..12]: [249, 258, 247, 243, 248, 244, 255, 255, 251, 248, 252, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688581988Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[0]R generation[0..12]: [136, 139, 134, 133, 135, 134, 138, 138, 136, 135, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688606438Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68863157Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[1]R flush_numbers[0..12]: [249, 258, 247, 243, 248, 244, 255, 255, 251, 248, 252, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688649316Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[1]R generation[0..12]: [136, 139, 134, 133, 135, 134, 138, 138, 136, 135, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688666161Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688683606Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[2]R flush_numbers[0..12]: [249, 258, 247, 243, 248, 244, 255, 255, 251, 248, 252, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688700781Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[2]R generation[0..12]: [136, 139, 134, 133, 135, 134, 138, 138, 136, 135, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688717425Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68873454Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688750643Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688766726Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688782258Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688798151Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"":"downstairs","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688814193Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688828914Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"0","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688845397Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"1","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68886139Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"client":"2","":"downstairs","session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68888072Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.68889572Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"f4928bdb-72a5-44f9-a1ce-a09befbc8d3d is now active with session: 26c4e620-a31e-490e-9f85-77be580fbd89","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688911533Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688926283Z","hostname":"bmat-EVT22200007-0000589c","pid":1747,"session_id":"26c4e620-a31e-490e-9f85-77be580fbd89"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688941835Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} {"msg":"Activated sub_volume f4928bdb-72a5-44f9-a1ce-a09befbc8d3d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:02.688956074Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 133 len 8192 data: 9 10 02/30 Write block 574 len 20480 data: 9 8 9 9 9 03/30 Read block 391 len 36864 04/30 Write block 499 len 16384 data: 8 10 11 11 05/30 Read block 545 len 28672 06/30 Write block 472 len 40960 data: 7 6 4 5 8 9 10 10 11 11 07/30 Read block 578 len 40960 08/30 Write block 139 len 20480 data: 14 13 13 11 16 09/30 Write block 552 len 24576 data: 12 12 14 12 13 13 10/30 Read block 415 len 4096 11/30 Write block 232 len 4096 data: 13 12/30 Write block 403 len 16384 data: 12 15 16 14 13/30 Write block 318 len 36864 data: 12 13 15 13 12 8 9 10 8 14/30 Read block 77 len 32768 15/30 Write block 77 len 36864 data: 8 12 12 14 13 12 11 11 11 16/30 Write block 538 len 16384 data: 12 12 11 10 17/30 Write block 306 len 20480 data: 8 8 6 6 8 18/30 Write block 510 len 16384 data: 15 14 12 12 19/30 Read block 39 len 4096 20/30 Read block 197 len 16384 21/30 Read block 564 len 28672 22/30 Write block 393 len 8192 data: 17 16 23/30 Write block 199 len 16384 data: 10 10 10 11 24/30 Write block 103 len 32768 data: 10 11 12 12 11 13 13 15 25/30 Read block 91 len 40960 26/30 Write block 65 len 16384 data: 8 9 10 9 27/30 Write block 207 len 24576 data: 9 9 8 9 6 7 28/30 Read block 256 len 8192 29/30 Read block 206 len 32768 30/30 Write block 542 len 28672 data: 11 10 10 6 10 11 12 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:02.713924017Z","hostname":"bmat-EVT22200007-0000589c","pid":1747} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 139 139 139 258 258 258 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 133 133 133 243 243 243 T T T 4 080-099 135 135 135 248 248 248 T T T 5 100-119 134 134 134 244 244 244 T T T 6 120-139 138 138 138 255 255 255 T T T 7 140-159 138 138 138 255 255 255 T T T 8 160-179 136 136 136 251 251 251 F F F 9 180-199 135 135 135 248 248 248 T T T 10 200-219 137 137 137 252 252 252 T T T 11 220-239 139 139 139 258 258 258 T T T 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 138 138 138 255 255 255 T T T 16 320-339 134 134 134 247 247 247 T T T 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 138 138 138 257 257 257 T T T 20 400-419 136 136 136 249 249 249 T T T 21 420-439 139 139 139 258 258 258 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 127 127 127 231 231 231 T T T 24 480-499 136 136 136 249 249 249 T T T 25 500-519 136 136 136 249 249 249 T T T 26 520-539 134 134 134 245 245 245 T T T 27 540-559 138 138 138 254 254 254 T T T 28 560-579 131 131 131 240 240 240 T T T 29 580-599 134 134 134 245 245 245 F F F Max gen: 139, Max flush: 258 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 139 139 139 258 258 258 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 140 140 140 259 259 259 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 140 140 140 259 259 259 F F F 7 140-159 140 140 140 259 259 259 F F F 8 160-179 136 136 136 251 251 251 F F F 9 180-199 140 140 140 259 259 259 F F F 10 200-219 140 140 140 259 259 259 F F F 11 220-239 140 140 140 259 259 259 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 139 139 139 258 258 258 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 140 140 140 259 259 259 F F F 25 500-519 140 140 140 259 259 259 F F F 26 520-539 140 140 140 259 259 259 F F F 27 540-559 140 140 140 259 259 259 F F F 28 560-579 140 140 140 259 259 259 F F F 29 580-599 134 134 134 245 245 245 F F F Max gen: 140, Max flush: 259 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.138111042Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.193430062Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.194200097Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.194272906Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.194303178Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.194330975Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"Crucible b9566145-a32f-44af-a448-dc3dd8858321 has session id: 71c980d4-a031-4e31-8a6d-85116eb78e19","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.195184712Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Upstairs opts: Upstairs UUID: b9566145-a32f-44af-a448-dc3dd8858321, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.195236989Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Crucible stats registered with UUID: b9566145-a32f-44af-a448-dc3dd8858321","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.195256779Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:17.195280618Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196196442Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196345878Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"b9566145-a32f-44af-a448-dc3dd8858321 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196367762Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196384006Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196569525Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196732609Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196758432Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196774895Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.19679205Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196807983Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196825489Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.196841993Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"ds_connection connected from Ok(127.0.0.1:49646)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.197103818Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"ds_connection connected from Ok(127.0.0.1:42355)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.197467813Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"ds_connection connected from Ok(127.0.0.1:35453)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.197589752Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.198110488Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.198210653Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.198412676Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.198823877Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.199049208Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.199263918Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.199611208Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.199879838Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[0]R flush_numbers[0..12]: [249, 258, 247, 259, 259, 259, 259, 259, 251, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.199988761Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[0]R generation[0..12]: [136, 139, 134, 140, 140, 140, 140, 140, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200014834Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200033713Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[1]R flush_numbers[0..12]: [249, 258, 247, 259, 259, 259, 259, 259, 251, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200050727Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[1]R generation[0..12]: [136, 139, 134, 140, 140, 140, 140, 140, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200067021Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200084166Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[2]R flush_numbers[0..12]: [249, 258, 247, 259, 259, 259, 259, 259, 251, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200107764Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[2]R generation[0..12]: [136, 139, 134, 140, 140, 140, 140, 140, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200124929Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200141734Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:09:17.200350992Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200384039Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200400673Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200416376Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"":"downstairs","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200432078Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200512482Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"0","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200607216Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"1","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:09:17.200688512Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"client":"2","":"downstairs","session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200799249Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200815202Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"b9566145-a32f-44af-a448-dc3dd8858321 is now active with session: 71c980d4-a031-4e31-8a6d-85116eb78e19","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200831154Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200845714Z","hostname":"bmat-EVT22200007-0000589c","pid":1755,"session_id":"71c980d4-a031-4e31-8a6d-85116eb78e19"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200865414Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} {"msg":"Activated sub_volume b9566145-a32f-44af-a448-dc3dd8858321","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:17.200879904Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 438 len 8192 data: 11 10 03/30 Read block 314 len 24576 04/30 Read block 582 len 16384 05/30 Read block 306 len 28672 06/30 Write block 587 len 40960 data: 12 11 12 9 11 9 6 7 6 6 07/30 Flush 08/30 Write block 497 len 8192 data: 7 7 09/30 Read block 410 len 28672 10/30 Read block 221 len 20480 11/30 Read block 285 len 40960 12/30 Write block 168 len 16384 data: 17 14 13 13 13/30 Flush 14/30 Flush 15/30 Write block 584 len 24576 data: 12 11 14 13 12 13 16/30 Read block 466 len 36864 17/30 Read block 585 len 24576 18/30 Write block 557 len 20480 data: 14 13 12 10 11 19/30 Flush 20/30 Flush 21/30 Write block 565 len 8192 data: 10 9 22/30 Read block 581 len 20480 23/30 Write block 594 len 4096 data: 8 24/30 Read block 535 len 32768 25/30 Write block 24 len 24576 data: 10 11 12 11 11 10 26/30 Write block 166 len 20480 data: 11 17 18 15 14 27/30 Write block 594 len 12288 data: 9 7 7 28/30 Write block 227 len 24576 data: 7 8 7 8 11 14 29/30 Read block 480 len 40960 30/30 Write block 145 len 40960 data: 17 15 13 13 11 11 10 6 4 5 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:17.22707107Z","hostname":"bmat-EVT22200007-0000589c","pid":1755} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 139 139 139 258 258 258 T T T 2 040-059 134 134 134 247 247 247 F F F 3 060-079 140 140 140 259 259 259 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 140 140 140 259 259 259 F F F 7 140-159 140 140 140 259 259 259 T T T 8 160-179 142 142 142 262 262 262 T T T 9 180-199 140 140 140 259 259 259 F F F 10 200-219 140 140 140 259 259 259 F F F 11 220-239 140 140 140 259 259 259 T T T 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 142 142 142 262 262 262 F F F 25 500-519 140 140 140 259 259 259 F F F 26 520-539 140 140 140 259 259 259 F F F 27 540-559 142 142 142 264 264 264 F F F 28 560-579 142 142 142 264 264 264 T T T 29 580-599 142 142 142 264 264 264 T T T Max gen: 142, Max flush: 264 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 F F F 1 020-039 143 143 143 265 265 265 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 140 140 140 259 259 259 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 140 140 140 259 259 259 F F F 7 140-159 143 143 143 265 265 265 F F F 8 160-179 143 143 143 265 265 265 F F F 9 180-199 140 140 140 259 259 259 F F F 10 200-219 140 140 140 259 259 259 F F F 11 220-239 143 143 143 265 265 265 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 136 136 136 250 250 250 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 142 142 142 262 262 262 F F F 25 500-519 140 140 140 259 259 259 F F F 26 520-539 140 140 140 259 259 259 F F F 27 540-559 142 142 142 264 264 264 F F F 28 560-579 143 143 143 265 265 265 F F F 29 580-599 143 143 143 265 265 265 F F F Max gen: 143, Max flush: 265 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.566749561Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.621684425Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.622494272Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.622548613Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.622569065Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.622592804Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"Crucible 5b6e22da-815b-4de0-b095-5d73f5ceae96 has session id: 230ae1c3-fda3-4769-9d88-0ea391b493a0","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.623671641Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Upstairs opts: Upstairs UUID: 5b6e22da-815b-4de0-b095-5d73f5ceae96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.623723687Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Crucible stats registered with UUID: 5b6e22da-815b-4de0-b095-5d73f5ceae96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.623743548Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:21.62375924Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624582775Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624611985Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"5b6e22da-815b-4de0-b095-5d73f5ceae96 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.62483455Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624857116Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.62488817Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624906948Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624924274Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624940256Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.624957822Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625018807Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625043047Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625059661Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34760)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625509271Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40350)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625598283Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58702)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.625632242Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.626574771Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.62660893Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.626629573Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.627348494Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.627518812Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.627544314Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628064118Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628396327Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[0]R flush_numbers[0..12]: [249, 265, 247, 259, 259, 259, 259, 265, 265, 259, 259, 265]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628567017Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[0]R generation[0..12]: [136, 143, 134, 140, 140, 140, 140, 143, 143, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628597309Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628615726Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[1]R flush_numbers[0..12]: [249, 265, 247, 259, 259, 259, 259, 265, 265, 259, 259, 265]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628632851Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[1]R generation[0..12]: [136, 143, 134, 140, 140, 140, 140, 143, 143, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628649576Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628666811Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[2]R flush_numbers[0..12]: [249, 265, 247, 259, 259, 259, 259, 265, 265, 259, 259, 265]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628684337Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[2]R generation[0..12]: [136, 143, 134, 140, 140, 140, 140, 143, 143, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628701462Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628718727Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628734459Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628750562Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628766224Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628782077Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"":"downstairs","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.62879822Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.62881298Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"0","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628831398Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"1","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628852982Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"client":"2","":"downstairs","session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628869726Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628884857Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"5b6e22da-815b-4de0-b095-5d73f5ceae96 is now active with session: 230ae1c3-fda3-4769-9d88-0ea391b493a0","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628900559Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628915269Z","hostname":"bmat-EVT22200007-0000589c","pid":1763,"session_id":"230ae1c3-fda3-4769-9d88-0ea391b493a0"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:09:21.62893025Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} {"msg":"Activated sub_volume 5b6e22da-815b-4de0-b095-5d73f5ceae96","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:21.628954519Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 36864 data: 16 16 10 10 7 8 10 12 14 02/30 Read block 220 len 28672 03/30 Write block 364 len 36864 data: 14 17 17 11 11 8 9 11 13 04/30 Read block 186 len 16384 05/30 Read block 36 len 16384 06/30 Read block 566 len 32768 07/30 Read block 544 len 8192 08/30 Write block 595 len 4096 data: 8 09/30 Write block 195 len 40960 data: 16 15 14 13 11 11 11 12 12 13 10/30 Write block 1 len 16384 data: 4 7 9 9 11/30 Read block 409 len 36864 12/30 Write block 35 len 16384 data: 16 14 15 17 13/30 Read block 229 len 24576 14/30 Write block 138 len 32768 data: 12 15 14 14 12 17 18 18 15/30 Read block 53 len 20480 16/30 Write block 552 len 12288 data: 13 13 15 17/30 Flush 18/30 Write block 12 len 28672 data: 10 10 11 8 5 5 5 19/30 Write block 237 len 8192 data: 13 13 20/30 Read block 77 len 4096 21/30 Write block 529 len 24576 data: 10 10 10 8 7 9 22/30 Read block 287 len 32768 23/30 Read block 520 len 12288 24/30 Read block 307 len 16384 25/30 Read block 416 len 28672 26/30 Read block 358 len 36864 27/30 Write block 516 len 20480 data: 11 11 9 6 6 28/30 Read block 566 len 24576 29/30 Read block 558 len 8192 30/30 Write block 148 len 24576 data: 14 12 12 11 7 5 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:21.65714707Z","hostname":"bmat-EVT22200007-0000589c","pid":1763} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 266 266 266 T T T 1 020-039 144 144 144 266 266 266 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 140 140 140 259 259 259 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 144 144 144 266 266 266 F F F 7 140-159 144 144 144 266 266 266 T T T 8 160-179 143 143 143 265 265 265 F F F 9 180-199 144 144 144 266 266 266 F F F 10 200-219 144 144 144 266 266 266 F F F 11 220-239 143 143 143 265 265 265 T T T 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 144 144 144 266 266 266 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 142 142 142 262 262 262 F F F 25 500-519 140 140 140 259 259 259 T T T 26 520-539 140 140 140 259 259 259 T T T 27 540-559 144 144 144 266 266 266 F F F 28 560-579 143 143 143 265 265 265 F F F 29 580-599 144 144 144 266 266 266 F F F Max gen: 144, Max flush: 266 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 267 267 267 F F F 1 020-039 144 144 144 266 266 266 F F F 2 040-059 134 134 134 247 247 247 F F F 3 060-079 140 140 140 259 259 259 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 144 144 144 266 266 266 F F F 7 140-159 145 145 145 267 267 267 F F F 8 160-179 143 143 143 265 265 265 F F F 9 180-199 144 144 144 266 266 266 F F F 10 200-219 144 144 144 266 266 266 F F F 11 220-239 145 145 145 267 267 267 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 138 138 138 255 255 255 F F F 14 280-299 138 138 138 255 255 255 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 140 140 140 259 259 259 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 144 144 144 266 266 266 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 142 142 142 262 262 262 F F F 25 500-519 145 145 145 267 267 267 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 144 144 144 266 266 266 F F F 28 560-579 143 143 143 265 265 265 F F F 29 580-599 144 144 144 266 266 266 F F F Max gen: 145, Max flush: 267 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:25.969695184Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.02467496Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.025592708Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.025651007Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.025670918Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.025688714Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"Crucible fd504b35-b51b-4b25-88e0-27aabf2b7fa9 has session id: 62322a4b-fb3b-4273-8387-c15a8bb2dc47","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.026688209Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Upstairs opts: Upstairs UUID: fd504b35-b51b-4b25-88e0-27aabf2b7fa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.026738231Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Crucible stats registered with UUID: fd504b35-b51b-4b25-88e0-27aabf2b7fa9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.026756769Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:26.02677168Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027444697Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027479167Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"fd504b35-b51b-4b25-88e0-27aabf2b7fa9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027616729Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027641399Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027674747Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027691692Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027709188Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027724629Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027741454Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027756895Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027853804Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.027876961Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"ds_connection connected from Ok(127.0.0.1:56660)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.028245866Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:26.028314126Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:26.028337413Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"ds_connection connected from Ok(127.0.0.1:49574)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.028356041Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:26.028375722Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.028397316Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.02891239Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.028949495Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.029722186Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.02985614Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.030423672Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:26.030535571Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.03024413Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"ds_connection connected from Ok(127.0.0.1:58979)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.03052303Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.03209198Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.032921608Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[0]R flush_numbers[0..12]: [267, 266, 247, 259, 259, 259, 266, 267, 265, 266, 266, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033816168Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[0]R generation[0..12]: [145, 144, 134, 140, 140, 140, 144, 145, 143, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033840047Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033857853Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[1]R flush_numbers[0..12]: [267, 266, 247, 259, 259, 259, 266, 267, 265, 266, 266, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033885369Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[1]R generation[0..12]: [145, 144, 134, 140, 140, 140, 144, 145, 143, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033902775Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033920531Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[2]R flush_numbers[0..12]: [267, 266, 247, 259, 259, 259, 266, 267, 265, 266, 266, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033936975Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[2]R generation[0..12]: [145, 144, 134, 140, 140, 140, 144, 145, 143, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033953318Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033969882Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.033985324Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034001146Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034016888Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034033011Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"":"downstairs","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034048553Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034062822Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"0","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034078615Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"1","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034094106Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"client":"2","":"downstairs","session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034109809Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034124168Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"fd504b35-b51b-4b25-88e0-27aabf2b7fa9 is now active with session: 62322a4b-fb3b-4273-8387-c15a8bb2dc47","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034145762Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034160312Z","hostname":"bmat-EVT22200007-0000589c","pid":1771,"session_id":"62322a4b-fb3b-4273-8387-c15a8bb2dc47"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034175192Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} {"msg":"Activated sub_volume fd504b35-b51b-4b25-88e0-27aabf2b7fa9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:36.034251408Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 16384 data: 14 12 13 12 02/30 Write block 588 len 4096 data: 13 03/30 Write block 273 len 36864 data: 8 10 10 10 10 10 13 12 10 04/30 Read block 228 len 12288 05/30 Read block 281 len 24576 06/30 Read block 75 len 20480 07/30 Read block 144 len 24576 08/30 Write block 217 len 24576 data: 9 8 8 10 9 13 09/30 Read block 457 len 12288 10/30 Read block 336 len 4096 11/30 Write block 320 len 8192 data: 16 14 12/30 Read block 514 len 8192 13/30 Write block 360 len 16384 data: 13 12 10 12 14/30 Write block 192 len 24576 data: 15 10 13 17 16 15 15/30 Write block 375 len 8192 data: 17 18 16/30 Read block 528 len 32768 17/30 Write block 36 len 24576 data: 15 16 18 15 15 15 18/30 Read block 227 len 28672 19/30 Write block 486 len 4096 data: 10 20/30 Read block 535 len 20480 21/30 Read block 95 len 16384 22/30 Write block 592 len 16384 data: 10 7 10 9 23/30 Write block 501 len 28672 data: 12 12 12 13 12 14 14 24/30 Flush 25/30 Write block 7 len 28672 data: 15 13 12 9 7 11 11 26/30 Read block 177 len 36864 27/30 Read block 20 len 16384 28/30 Write block 47 len 36864 data: 12 9 11 12 11 16 18 17 15 29/30 Read block 426 len 16384 30/30 Write block 62 len 28672 data: 10 8 7 9 10 11 10 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:36.063855988Z","hostname":"bmat-EVT22200007-0000589c","pid":1771} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 267 267 267 T T T 1 020-039 146 146 146 268 268 268 F F F 2 040-059 146 146 146 268 268 268 T T T 3 060-079 140 140 140 259 259 259 T T T 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 144 144 144 266 266 266 F F F 7 140-159 145 145 145 267 267 267 F F F 8 160-179 146 146 146 268 268 268 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 146 146 146 268 268 268 F F F 11 220-239 146 146 146 268 268 268 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 146 146 146 268 268 268 F F F 14 280-299 146 146 146 268 268 268 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 146 146 146 268 268 268 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 146 146 146 268 268 268 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 146 146 146 268 268 268 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 144 144 144 266 266 266 F F F 28 560-579 143 143 143 265 265 265 F F F 29 580-599 146 146 146 268 268 268 F F F Max gen: 146, Max flush: 268 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 269 269 269 F F F 1 020-039 146 146 146 268 268 268 F F F 2 040-059 147 147 147 269 269 269 F F F 3 060-079 147 147 147 269 269 269 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 144 144 144 266 266 266 F F F 7 140-159 145 145 145 267 267 267 F F F 8 160-179 146 146 146 268 268 268 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 146 146 146 268 268 268 F F F 11 220-239 146 146 146 268 268 268 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 146 146 146 268 268 268 F F F 14 280-299 146 146 146 268 268 268 F F F 15 300-319 140 140 140 259 259 259 F F F 16 320-339 146 146 146 268 268 268 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 146 146 146 268 268 268 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 146 146 146 268 268 268 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 144 144 144 266 266 266 F F F 28 560-579 143 143 143 265 265 265 F F F 29 580-599 146 146 146 268 268 268 F F F Max gen: 147, Max flush: 269 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.373891693Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.428033855Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.428875587Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.428938195Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.428958286Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.428976714Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"Crucible 78a8dbd1-3c10-4c1c-a513-76ccd5be591f has session id: f766a37f-08f4-4e64-970f-2528eb1062e7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.43012328Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Upstairs opts: Upstairs UUID: 78a8dbd1-3c10-4c1c-a513-76ccd5be591f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.430175216Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Crucible stats registered with UUID: 78a8dbd1-3c10-4c1c-a513-76ccd5be591f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.430195247Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:40.430211049Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.430929309Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.430957747Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"78a8dbd1-3c10-4c1c-a513-76ccd5be591f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.43107653Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431098746Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431129078Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431145952Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431163809Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431184601Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431202448Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431218581Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431372295Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431396084Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53185)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431413951Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38348)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431718043Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51790)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.431778306Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.432391811Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.432428005Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.432660912Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.433339941Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.433571385Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.43394552Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434127352Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434229591Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[0]R flush_numbers[0..12]: [269, 268, 269, 269, 259, 259, 266, 267, 268, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.43456607Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[0]R generation[0..12]: [147, 146, 147, 147, 140, 140, 144, 145, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434603707Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434622595Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[1]R flush_numbers[0..12]: [269, 268, 269, 269, 259, 259, 266, 267, 268, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434640221Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[1]R generation[0..12]: [147, 146, 147, 147, 140, 140, 144, 145, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434657517Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434674471Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[2]R flush_numbers[0..12]: [269, 268, 269, 269, 259, 259, 266, 267, 268, 268, 268, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434691627Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[2]R generation[0..12]: [147, 146, 147, 147, 140, 140, 144, 145, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434708541Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434725977Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434741779Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434757481Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434772863Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434788094Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"":"downstairs","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:09:40.434809929Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434838517Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"0","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434855362Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"1","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434870973Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"client":"2","":"downstairs","session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434886856Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434901295Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"78a8dbd1-3c10-4c1c-a513-76ccd5be591f is now active with session: f766a37f-08f4-4e64-970f-2528eb1062e7","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434916897Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434931066Z","hostname":"bmat-EVT22200007-0000589c","pid":1779,"session_id":"f766a37f-08f4-4e64-970f-2528eb1062e7"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434945526Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} {"msg":"Activated sub_volume 78a8dbd1-3c10-4c1c-a513-76ccd5be591f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:40.434959304Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 573 len 36864 data: 8 10 9 10 10 10 9 9 10 02/30 Read block 468 len 4096 03/30 Write block 368 len 8192 data: 12 9 04/30 Write block 131 len 32768 data: 9 9 10 11 11 11 11 13 05/30 Flush 06/30 Write block 15 len 4096 data: 9 07/30 Write block 554 len 16384 data: 16 13 14 15 08/30 Read block 584 len 12288 09/30 Read block 124 len 12288 10/30 Write block 563 len 8192 data: 7 8 11/30 Read block 47 len 40960 12/30 Write block 147 len 40960 data: 14 15 13 13 12 8 6 6 6 8 13/30 Read block 105 len 16384 14/30 Write block 168 len 40960 data: 19 16 15 15 13 14 13 12 10 9 15/30 Read block 68 len 8192 16/30 Read block 206 len 36864 17/30 Flush 18/30 Write block 365 len 40960 data: 18 18 12 13 10 10 12 14 15 19 19/30 Read block 30 len 36864 20/30 Read block 91 len 24576 21/30 Read block 100 len 28672 22/30 Write block 30 len 16384 data: 11 11 12 11 23/30 Read block 599 len 4096 24/30 Read block 131 len 8192 25/30 Write block 208 len 12288 data: 10 9 10 26/30 Write block 422 len 4096 data: 8 27/30 Read block 97 len 20480 28/30 Read block 535 len 36864 29/30 Write block 223 len 40960 data: 14 14 14 10 8 9 8 9 12 15 30/30 Write block 303 len 36864 data: 9 8 7 9 9 7 7 9 9 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:40.46496787Z","hostname":"bmat-EVT22200007-0000589c","pid":1779} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 271 271 271 F F F 1 020-039 146 146 146 268 268 268 T T T 2 040-059 147 147 147 269 269 269 F F F 3 060-079 147 147 147 269 269 269 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 148 148 148 270 270 270 F F F 7 140-159 148 148 148 271 271 271 F F F 8 160-179 148 148 148 271 271 271 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 146 146 146 268 268 268 T T T 11 220-239 146 146 146 268 268 268 T T T 12 240-259 138 138 138 257 257 257 F F F 13 260-279 146 146 146 268 268 268 F F F 14 280-299 146 146 146 268 268 268 F F F 15 300-319 140 140 140 259 259 259 T T T 16 320-339 146 146 146 268 268 268 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 148 148 148 270 270 270 T T T 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 142 142 142 261 261 261 T T T 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 146 146 146 268 268 268 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 148 148 148 271 271 271 F F F 28 560-579 148 148 148 271 271 271 F F F 29 580-599 148 148 148 270 270 270 F F F Max gen: 148, Max flush: 271 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 271 271 271 F F F 1 020-039 149 149 149 272 272 272 F F F 2 040-059 147 147 147 269 269 269 F F F 3 060-079 147 147 147 269 269 269 F F F 4 080-099 140 140 140 259 259 259 F F F 5 100-119 140 140 140 259 259 259 F F F 6 120-139 148 148 148 270 270 270 F F F 7 140-159 148 148 148 271 271 271 F F F 8 160-179 148 148 148 271 271 271 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 149 149 149 272 272 272 F F F 11 220-239 149 149 149 272 272 272 F F F 12 240-259 138 138 138 257 257 257 F F F 13 260-279 146 146 146 268 268 268 F F F 14 280-299 146 146 146 268 268 268 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 146 146 146 268 268 268 F F F 17 340-359 137 137 137 252 252 252 F F F 18 360-379 149 149 149 272 272 272 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 F F F 21 420-439 149 149 149 272 272 272 F F F 22 440-459 136 136 136 250 250 250 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 146 146 146 268 268 268 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 148 148 148 271 271 271 F F F 28 560-579 148 148 148 271 271 271 F F F 29 580-599 148 148 148 270 270 270 F F F Max gen: 149, Max flush: 272 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.796859731Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.850375873Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.851109203Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.85118109Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.85120081Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.851218867Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"Crucible 072fa867-d9bf-4976-b1e5-d73fa2ce632b has session id: 3061d078-2985-4706-acba-211692108e64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.852196578Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Upstairs opts: Upstairs UUID: 072fa867-d9bf-4976-b1e5-d73fa2ce632b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.852335722Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Crucible stats registered with UUID: 072fa867-d9bf-4976-b1e5-d73fa2ce632b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.852361996Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:44.852384452Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853136922Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853166763Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"072fa867-d9bf-4976-b1e5-d73fa2ce632b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853430302Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853452096Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853482879Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853499493Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853516578Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853532341Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853549215Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853727299Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853795399Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.85381541Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:45659)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853833437Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:43299)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853850061Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:61817)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.853914422Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.854733107Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.855148918Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.855777093Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.85601018Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.856148784Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.856697767Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.856846802Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.857061992Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[0]R flush_numbers[0..12]: [271, 272, 269, 269, 259, 259, 270, 271, 271, 268, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.857879034Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[0]R generation[0..12]: [148, 149, 147, 147, 140, 140, 148, 148, 148, 146, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.857941602Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.857976914Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[1]R flush_numbers[0..12]: [271, 272, 269, 269, 259, 259, 270, 271, 271, 268, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858009811Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[1]R generation[0..12]: [148, 149, 147, 147, 140, 140, 148, 148, 148, 146, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858043911Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.85807783Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[2]R flush_numbers[0..12]: [271, 272, 269, 269, 259, 259, 270, 271, 271, 268, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858122131Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[2]R generation[0..12]: [148, 149, 147, 147, 140, 140, 148, 148, 148, 146, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858156381Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858188326Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858222727Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858255133Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858284343Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858316529Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"":"downstairs","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858349226Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:09:44.858375489Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"0","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858426233Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"1","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858460082Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"client":"2","":"downstairs","session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858489803Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858518221Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"072fa867-d9bf-4976-b1e5-d73fa2ce632b is now active with session: 3061d078-2985-4706-acba-211692108e64","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858549125Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858576521Z","hostname":"bmat-EVT22200007-0000589c","pid":1787,"session_id":"3061d078-2985-4706-acba-211692108e64"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858606231Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} {"msg":"Activated sub_volume 072fa867-d9bf-4976-b1e5-d73fa2ce632b","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:44.858687648Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 36864 data: 11 11 10 10 11 10 11 13 12 02/30 Read block 84 len 32768 03/30 Flush 04/30 Write block 216 len 36864 data: 6 10 9 9 11 10 14 15 15 05/30 Write block 350 len 4096 data: 10 06/30 Read block 339 len 16384 07/30 Read block 76 len 36864 08/30 Write block 16 len 28672 data: 6 6 6 4 4 3 9 09/30 Read block 378 len 12288 10/30 Read block 539 len 40960 11/30 Read block 244 len 8192 12/30 Write block 61 len 36864 data: 13 11 9 8 10 11 12 11 9 13/30 Write block 103 len 12288 data: 11 12 13 14/30 Write block 503 len 24576 data: 13 14 13 15 15 15 15/30 Read block 114 len 32768 16/30 Write block 437 len 20480 data: 10 12 11 10 9 17/30 Write block 81 len 32768 data: 14 13 12 12 12 10 9 8 18/30 Write block 344 len 32768 data: 14 13 12 13 13 10 11 9 19/30 Write block 407 len 20480 data: 13 17 17 16 15 20/30 Read block 404 len 28672 21/30 Write block 439 len 40960 data: 12 11 10 10 9 11 11 13 12 14 22/30 Write block 274 len 32768 data: 11 11 11 11 11 14 13 11 23/30 Write block 168 len 36864 data: 20 17 16 16 14 15 14 13 11 24/30 Write block 337 len 36864 data: 13 12 13 13 14 18 13 15 14 25/30 Write block 218 len 24576 data: 10 10 12 11 15 16 26/30 Read block 237 len 40960 27/30 Write block 417 len 24576 data: 6 6 8 10 8 9 28/30 Write block 343 len 12288 data: 14 16 15 29/30 Write block 247 len 40960 data: 13 12 13 13 13 11 12 11 15 16 30/30 Write block 251 len 32768 data: 14 12 13 12 16 17 16 13 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 9 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 10 Sent Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 25 26 25 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:44.884456029Z","hostname":"bmat-EVT22200007-0000589c","pid":1787} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 271 271 271 T T T 1 020-039 149 149 149 272 272 272 T T T 2 040-059 147 147 147 269 269 269 F F F 3 060-079 147 147 147 269 269 269 T T T 4 080-099 140 140 140 259 259 259 T T T 5 100-119 140 140 140 259 259 259 T T T 6 120-139 148 148 148 270 270 270 F F F 7 140-159 148 148 148 271 271 271 F F F 8 160-179 148 148 148 271 271 271 T T T 9 180-199 146 146 146 268 268 268 F F F 10 200-219 149 149 149 272 272 272 T T T 11 220-239 149 149 149 272 272 272 T T T 12 240-259 138 138 138 257 257 257 T T T 13 260-279 146 146 146 268 268 268 T T T 14 280-299 146 146 146 268 268 268 T T T 15 300-319 149 149 149 272 272 272 F F F 16 320-339 146 146 146 268 268 268 T T T 17 340-359 137 137 137 252 252 252 T T T 18 360-379 149 149 149 272 272 272 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 140 140 140 259 259 259 T T T 21 420-439 149 149 149 272 272 272 T T T 22 440-459 136 136 136 250 250 250 T T T 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 146 146 146 268 268 268 T T T 26 520-539 145 145 145 267 267 267 F F F 27 540-559 148 148 148 271 271 271 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 150 150 150 273 273 273 F F F Max gen: 150, Max flush: 273 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 151 151 151 274 274 274 F F F 2 040-059 147 147 147 269 269 269 F F F 3 060-079 151 151 151 274 274 274 F F F 4 080-099 151 151 151 274 274 274 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 148 148 148 270 270 270 F F F 7 140-159 148 148 148 271 271 271 F F F 8 160-179 151 151 151 274 274 274 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 151 151 151 274 274 274 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 151 151 151 274 274 274 F F F 13 260-279 151 151 151 274 274 274 F F F 14 280-299 151 151 151 274 274 274 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 151 151 151 274 274 274 F F F 17 340-359 151 151 151 274 274 274 F F F 18 360-379 149 149 149 272 272 272 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 151 151 151 274 274 274 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 151 151 151 274 274 274 F F F 23 460-479 140 140 140 259 259 259 F F F 24 480-499 146 146 146 268 268 268 F F F 25 500-519 151 151 151 274 274 274 F F F 26 520-539 145 145 145 267 267 267 F F F 27 540-559 148 148 148 271 271 271 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 150 150 150 273 273 273 F F F Max gen: 151, Max flush: 274 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.297199368Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.35137085Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.352159995Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.352217793Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.352238625Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.352256572Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"Crucible 71d60574-27ca-45e0-8308-16d893df8b85 has session id: 83083b86-d434-49d6-b27a-50a615ae3da9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.353136062Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Upstairs opts: Upstairs UUID: 71d60574-27ca-45e0-8308-16d893df8b85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.353404882Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Crucible stats registered with UUID: 71d60574-27ca-45e0-8308-16d893df8b85","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.353427588Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:49.35344327Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.354146298Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.35417645Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"71d60574-27ca-45e0-8308-16d893df8b85 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.354307929Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.354333231Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355019325Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355089418Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.35511013Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355127205Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355144851Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355160714Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.35517821Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355194653Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50226)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355211277Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53661)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355228052Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55862)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355244275Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.355978277Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.356082269Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.356106038Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.357038606Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.357176067Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.357309741Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358054965Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358118876Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[0]R flush_numbers[0..12]: [274, 274, 269, 274, 274, 274, 270, 271, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358142314Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[0]R generation[0..12]: [151, 151, 147, 151, 151, 151, 148, 148, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358161974Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358180091Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[1]R flush_numbers[0..12]: [274, 274, 269, 274, 274, 274, 270, 271, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358197457Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[1]R generation[0..12]: [151, 151, 147, 151, 151, 151, 148, 148, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358214953Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358232298Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[2]R flush_numbers[0..12]: [274, 274, 269, 274, 274, 274, 270, 271, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358249554Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[2]R generation[0..12]: [151, 151, 147, 151, 151, 151, 148, 148, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358266749Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358284285Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358300368Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358323395Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358339468Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.35835535Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"":"downstairs","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358371613Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358386183Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"0","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358402877Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"1","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358495096Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"client":"2","":"downstairs","session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358517261Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358532683Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"71d60574-27ca-45e0-8308-16d893df8b85 is now active with session: 83083b86-d434-49d6-b27a-50a615ae3da9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358548435Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358562835Z","hostname":"bmat-EVT22200007-0000589c","pid":1795,"session_id":"83083b86-d434-49d6-b27a-50a615ae3da9"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358577595Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} {"msg":"Activated sub_volume 71d60574-27ca-45e0-8308-16d893df8b85","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:49.358591062Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 40960 data: 11 10 11 13 12 14 13 14 14 15 02/30 Write block 446 len 28672 data: 14 13 15 11 11 9 10 03/30 Flush 04/30 Write block 246 len 40960 data: 13 15 14 15 15 16 13 14 13 17 05/30 Write block 529 len 36864 data: 11 11 11 9 8 10 9 11 10 06/30 Read block 348 len 24576 07/30 Write block 492 len 32768 data: 9 11 7 6 6 8 8 9 08/30 Write block 495 len 12288 data: 7 7 9 09/30 Read block 285 len 4096 10/30 Write block 362 len 28672 data: 11 13 15 19 19 13 14 11/30 Write block 447 len 32768 data: 14 16 12 12 10 11 10 8 12/30 Read block 70 len 32768 13/30 Read block 571 len 40960 14/30 Flush 15/30 Write block 547 len 28672 data: 12 13 12 11 10 14 14 16/30 Read block 274 len 4096 17/30 Read block 254 len 8192 18/30 Write block 139 len 40960 data: 16 15 15 13 18 19 19 16 15 16 19/30 Read block 468 len 12288 20/30 Read block 179 len 40960 21/30 Read block 572 len 8192 22/30 Read block 344 len 4096 23/30 Write block 79 len 28672 data: 13 15 15 14 13 13 13 24/30 Flush 25/30 Write block 453 len 36864 data: 11 9 8 10 9 10 10 8 9 26/30 Read block 537 len 28672 27/30 Write block 34 len 32768 data: 17 17 16 17 19 16 16 16 28/30 Write block 375 len 4096 data: 18 29/30 Write block 284 len 8192 data: 8 7 30/30 Write block 263 len 16384 data: 10 9 11 12 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:49.387384172Z","hostname":"bmat-EVT22200007-0000589c","pid":1795} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 151 151 151 274 274 274 T T T 2 040-059 147 147 147 269 269 269 T T T 3 060-079 152 152 152 277 277 277 F F F 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 151 151 151 274 274 274 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 151 151 151 274 274 274 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 151 151 151 274 274 274 T T T 14 280-299 151 151 151 274 274 274 T T T 15 300-319 149 149 149 272 272 272 F F F 16 320-339 151 151 151 274 274 274 F F F 17 340-359 151 151 151 274 274 274 F F F 18 360-379 152 152 152 276 276 276 T T T 19 380-399 140 140 140 259 259 259 F F F 20 400-419 151 151 151 274 274 274 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 152 152 152 276 276 276 T T T 23 460-479 140 140 140 259 259 259 T T T 24 480-499 152 152 152 276 276 276 F F F 25 500-519 151 151 151 274 274 274 F F F 26 520-539 152 152 152 276 276 276 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 150 150 150 273 273 273 F F F Max gen: 152, Max flush: 277 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 153 153 153 278 278 278 F F F 2 040-059 153 153 153 278 278 278 F F F 3 060-079 152 152 152 277 277 277 F F F 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 151 151 151 274 274 274 F F F 9 180-199 146 146 146 268 268 268 F F F 10 200-219 151 151 151 274 274 274 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 153 153 153 278 278 278 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 151 151 151 274 274 274 F F F 17 340-359 151 151 151 274 274 274 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 151 151 151 274 274 274 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 153 153 153 278 278 278 F F F 23 460-479 153 153 153 278 278 278 F F F 24 480-499 152 152 152 276 276 276 F F F 25 500-519 151 151 151 274 274 274 F F F 26 520-539 152 152 152 276 276 276 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 150 150 150 273 273 273 F F F Max gen: 153, Max flush: 278 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.724114511Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:53.777950766Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.778434235Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.815963248Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.816649783Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.816710326Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.816737252Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.816766762Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"Crucible 3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04 has session id: d542fd4b-b1d9-479a-b855-16953ea47282","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.818136975Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Upstairs opts: Upstairs UUID: 3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.818235236Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Crucible stats registered with UUID: 3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.818257281Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:53.818273284Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.818925549Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.818962795Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819077229Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819099575Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819134426Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819151491Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819169388Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819185941Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819203708Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.81921933Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819236455Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819327912Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"ds_connection connected from Ok(127.0.0.1:45621)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819461816Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"ds_connection connected from Ok(127.0.0.1:36409)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819487188Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"ds_connection connected from Ok(127.0.0.1:54218)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.819504884Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.820751034Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.820835817Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.820870087Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.821462549Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.821659432Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.821693622Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.82221087Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822319042Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[0]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822476975Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[0]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822500803Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822518119Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[1]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822541978Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[1]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822559143Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822575617Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[2]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 274, 268, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822592491Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[2]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 151, 146, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822608684Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822625328Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822640549Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822656031Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822671252Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822686604Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"":"downstairs","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822702296Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822716625Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"0","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822732157Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"1","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822747298Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"client":"2","":"downstairs","session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.82276284Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822780856Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04 is now active with session: d542fd4b-b1d9-479a-b855-16953ea47282","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822796398Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822810838Z","hostname":"bmat-EVT22200007-0000589c","pid":1803,"session_id":"d542fd4b-b1d9-479a-b855-16953ea47282"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822825137Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} {"msg":"Activated sub_volume 3db4ad6d-7da6-4e2c-b9cd-09e80a1ffd04","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:53.822839005Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 40960 data: 14 15 14 16 16 16 14 16 15 13 02/30 Write block 348 len 16384 data: 14 11 12 10 03/30 Write block 329 len 28672 data: 12 15 15 15 16 15 12 04/30 Read block 244 len 40960 05/30 Read block 222 len 28672 06/30 Write block 196 len 36864 data: 17 16 14 12 12 12 13 13 14 07/30 Flush 08/30 Write block 193 len 8192 data: 11 14 09/30 Read block 489 len 4096 10/30 Write block 580 len 4096 data: 11 11/30 Read block 401 len 24576 12/30 Read block 340 len 40960 13/30 Read block 279 len 20480 14/30 Read block 16 len 24576 15/30 Write block 493 len 36864 data: 12 8 8 8 10 9 10 11 13 16/30 Read block 252 len 36864 17/30 Write block 351 len 36864 data: 11 11 9 9 11 13 15 15 16 18/30 Read block 364 len 20480 19/30 Write block 476 len 20480 data: 9 10 11 11 12 20/30 Read block 21 len 16384 21/30 Write block 212 len 8192 data: 8 6 22/30 Write block 524 len 32768 data: 7 8 10 11 12 12 12 12 23/30 Flush 24/30 Write block 170 len 40960 data: 17 17 15 16 15 14 12 10 11 12 25/30 Read block 204 len 40960 26/30 Read block 340 len 8192 27/30 Read block 5 len 20480 28/30 Read block 582 len 24576 29/30 Read block 482 len 32768 30/30 Read block 123 len 20480 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:53.852872132Z","hostname":"bmat-EVT22200007-0000589c","pid":1803} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 153 153 153 278 278 278 F F F 2 040-059 153 153 153 278 278 278 F F F 3 060-079 152 152 152 277 277 277 F F F 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 151 151 151 274 274 274 T T T 9 180-199 154 154 154 280 280 280 F F F 10 200-219 154 154 154 280 280 280 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 153 153 153 278 278 278 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 154 154 154 280 280 280 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 151 151 151 274 274 274 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 153 153 153 278 278 278 F F F 23 460-479 154 154 154 280 280 280 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 154 154 154 280 280 280 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 154 154 154 280 280 280 F F F Max gen: 154, Max flush: 280 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 153 153 153 278 278 278 F F F 2 040-059 153 153 153 278 278 278 F F F 3 060-079 152 152 152 277 277 277 F F F 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 155 155 155 281 281 281 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 154 154 154 280 280 280 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 153 153 153 278 278 278 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 154 154 154 280 280 280 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 140 140 140 259 259 259 F F F 20 400-419 151 151 151 274 274 274 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 153 153 153 278 278 278 F F F 23 460-479 154 154 154 280 280 280 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 154 154 154 280 280 280 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 154 154 154 280 280 280 F F F Max gen: 155, Max flush: 281 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.181354106Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:58.233189278Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.23362059Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.27001411Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.270668408Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.270712148Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.270729453Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.270746628Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"Crucible 5d33415f-b337-4e1f-87b8-66e7b9538e2f has session id: 5bec23f2-9d2a-45de-ac41-98a906018f74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.271699738Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Upstairs opts: Upstairs UUID: 5d33415f-b337-4e1f-87b8-66e7b9538e2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.271792218Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Crucible stats registered with UUID: 5d33415f-b337-4e1f-87b8-66e7b9538e2f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.271814684Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:58.271829885Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272613247Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272645012Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"5d33415f-b337-4e1f-87b8-66e7b9538e2f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272774678Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272796843Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272828297Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272852497Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272870744Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272886356Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.272975028Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.273135095Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.27316182Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.273178724Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"ds_connection connected from Ok(127.0.0.1:54823)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.273195689Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"ds_connection connected from Ok(127.0.0.1:34291)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.273212042Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"ds_connection connected from Ok(127.0.0.1:38458)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.273414707Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.274330922Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.274388349Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.274407258Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.275382543Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.275600189Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.275631352Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276044818Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276208062Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[0]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 281, 280, 280, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276318879Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[0]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 155, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276343108Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276362057Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[1]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 281, 280, 280, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276379042Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[1]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 155, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276396297Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276413342Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[2]R flush_numbers[0..12]: [274, 278, 278, 277, 277, 274, 277, 277, 281, 280, 280, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276430377Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[2]R generation[0..12]: [151, 153, 153, 152, 152, 151, 152, 152, 155, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276446771Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276463395Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276478846Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276502495Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276518387Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276533839Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"":"downstairs","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.2765493Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.27656388Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"0","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276580314Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"1","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276595695Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"client":"2","":"downstairs","session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276611628Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276625977Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"Wait for a query_work_queue command to finish before sending IO msg":"5d33415f-b337-4e1f-87b8-66e7b9538e2f is now active with session: 5bec23f2-9d2a-45de-ac41-98a906018f74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276653864Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276669376Z","hostname":"bmat-EVT22200007-0000589c","pid":1811,"session_id":"5bec23f2-9d2a-45de-ac41-98a906018f74"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276683775Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} {"msg":"Activated sub_volume 5d33415f-b337-4e1f-87b8-66e7b9538e2f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:09:58.276697153Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 32768 data: 14 15 13 13 10 11 10 11 02/30 Read block 463 len 16384 03/30 Read block 115 len 32768 04/30 Write block 44 len 16384 data: 12 12 12 13 05/30 Write block 397 len 40960 data: 9 9 9 12 15 12 13 16 17 15 06/30 Read block 6 len 16384 07/30 Flush 08/30 Write block 346 len 40960 data: 13 14 15 12 13 12 12 10 10 12 09/30 Read block 394 len 28672 10/30 Read block 421 len 12288 11/30 Read block 421 len 28672 12/30 Write block 403 len 12288 data: 14 17 18 13/30 Write block 276 len 4096 data: 12 14/30 Read block 500 len 24576 15/30 Write block 442 len 12288 data: 11 10 12 16/30 Write block 70 len 20480 data: 8 8 9 8 9 17/30 Read block 32 len 32768 18/30 Read block 428 len 40960 19/30 Flush 20/30 Write block 344 len 36864 data: 17 16 14 15 16 13 14 13 13 21/30 Read block 403 len 8192 22/30 Read block 116 len 16384 23/30 Flush 24/30 Write block 57 len 28672 data: 16 14 13 16 14 12 10 25/30 Read block 420 len 36864 26/30 Read block 391 len 8192 27/30 Write block 421 len 24576 data: 9 10 8 8 7 5 28/30 Read block 543 len 24576 29/30 Read block 353 len 8192 30/30 Write block 33 len 12288 data: 12 18 18 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:09:58.306582517Z","hostname":"bmat-EVT22200007-0000589c","pid":1811} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 153 153 153 278 278 278 T T T 2 040-059 156 156 156 282 282 282 T T T 3 060-079 156 156 156 283 283 283 T T T 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 155 155 155 281 281 281 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 156 156 156 282 282 282 F F F 20 400-419 156 156 156 283 283 283 F F F 21 420-439 151 151 151 274 274 274 T T T 22 440-459 156 156 156 283 283 283 F F F 23 460-479 154 154 154 280 280 280 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 154 154 154 280 280 280 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 154 154 154 280 280 280 F F F Max gen: 156, Max flush: 284 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 152 152 152 277 277 277 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 152 152 152 277 277 277 F F F 8 160-179 155 155 155 281 281 281 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 149 149 149 272 272 272 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 156 156 156 282 282 282 F F F 20 400-419 156 156 156 283 283 283 F F F 21 420-439 157 157 157 285 285 285 F F F 22 440-459 156 156 156 283 283 283 F F F 23 460-479 154 154 154 280 280 280 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 154 154 154 280 280 280 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 152 152 152 277 277 277 F F F 28 560-579 150 150 150 273 273 273 F F F 29 580-599 154 154 154 280 280 280 F F F Max gen: 157, Max flush: 285 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.657276806Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.71102867Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.711777692Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.711829789Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.711848327Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.711867807Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"Crucible 02dffa61-6f6c-4033-b4ee-53056df09d26 has session id: a0802f81-d993-44ce-a939-65451da2553f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713005254Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Upstairs opts: Upstairs UUID: 02dffa61-6f6c-4033-b4ee-53056df09d26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713064295Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Crucible stats registered with UUID: 02dffa61-6f6c-4033-b4ee-53056df09d26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713092352Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:02.713108876Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713563777Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.71358969Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"02dffa61-6f6c-4033-b4ee-53056df09d26 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713694003Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713714645Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713749316Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713768285Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713786652Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713803076Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713821855Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713838018Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.713855103Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714197523Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55832)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714234739Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54627)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714255121Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55198)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714281154Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714737738Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.714922005Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.715460227Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.715683784Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.71574536Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.716223799Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.716304895Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.716719353Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[0]R flush_numbers[0..12]: [274, 285, 285, 285, 277, 274, 277, 277, 281, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717544862Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[0]R generation[0..12]: [151, 157, 157, 157, 152, 151, 152, 152, 155, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717571576Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717591136Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[1]R flush_numbers[0..12]: [274, 285, 285, 285, 277, 274, 277, 277, 281, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717609504Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[1]R generation[0..12]: [151, 157, 157, 157, 152, 151, 152, 152, 155, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717625867Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717650909Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[2]R flush_numbers[0..12]: [274, 285, 285, 285, 277, 274, 277, 277, 281, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717667893Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[2]R generation[0..12]: [151, 157, 157, 157, 152, 151, 152, 152, 155, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717683936Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717703276Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717718948Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:10:02.717845827Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.717875749Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:10:02.718157114Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"":"downstairs","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718296159Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718314456Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"0","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718331411Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"1","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718347824Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"client":"2","":"downstairs","session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718364268Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718379108Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"02dffa61-6f6c-4033-b4ee-53056df09d26 is now active with session: a0802f81-d993-44ce-a939-65451da2553f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718395382Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718418549Z","hostname":"bmat-EVT22200007-0000589c","pid":1819,"session_id":"a0802f81-d993-44ce-a939-65451da2553f"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.718434432Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} {"msg":"Activated sub_volume 02dffa61-6f6c-4033-b4ee-53056df09d26","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:02.71844839Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 28672 data: 7 7 9 11 10 11 9 02/30 Write block 300 len 20480 data: 12 10 10 10 9 03/30 Read block 364 len 24576 04/30 Flush 05/30 Write block 7 len 24576 data: 16 14 13 10 8 12 06/30 Read block 473 len 4096 07/30 Write block 94 len 20480 data: 12 14 16 17 15 08/30 Flush 09/30 Write block 450 len 4096 data: 13 10/30 Flush 11/30 Write block 514 len 16384 data: 12 12 12 12 12/30 Flush 13/30 Write block 500 len 12288 data: 12 14 13 14/30 Read block 494 len 16384 15/30 Read block 324 len 24576 16/30 Read block 536 len 40960 17/30 Read block 202 len 32768 18/30 Write block 151 len 24576 data: 13 9 7 7 7 9 19/30 Read block 521 len 8192 20/30 Write block 174 len 12288 data: 16 15 13 21/30 Read block 408 len 4096 22/30 Read block 494 len 24576 23/30 Write block 518 len 8192 data: 10 7 24/30 Write block 552 len 28672 data: 15 15 17 14 15 16 14 25/30 Flush 26/30 Write block 556 len 32768 data: 16 17 15 13 11 12 11 8 27/30 Write block 581 len 40960 data: 12 11 12 14 13 15 14 14 14 10 28/30 Read block 163 len 12288 29/30 Write block 472 len 8192 data: 8 7 30/30 Write block 399 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:02.744985444Z","hostname":"bmat-EVT22200007-0000589c","pid":1819} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 158 158 158 287 287 287 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 158 158 158 290 290 290 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 158 158 158 286 286 286 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 156 156 156 282 282 282 T T T 20 400-419 158 158 158 286 286 286 F F F 21 420-439 158 158 158 286 286 286 F F F 22 440-459 158 158 158 288 288 288 F F F 23 460-479 154 154 154 280 280 280 T T T 24 480-499 154 154 154 280 280 280 F F F 25 500-519 158 158 158 290 290 290 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 158 158 158 290 290 290 T T T 28 560-579 150 150 150 273 273 273 T T T 29 580-599 154 154 154 280 280 280 T T T Max gen: 158, Max flush: 290 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 158 158 158 287 287 287 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 158 158 158 290 290 290 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 158 158 158 286 286 286 F F F 16 320-339 154 154 154 279 279 279 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 159 159 159 291 291 291 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 158 158 158 286 286 286 F F F 22 440-459 158 158 158 288 288 288 F F F 23 460-479 159 159 159 291 291 291 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 158 158 158 290 290 290 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 159 159 159 291 291 291 F F F 28 560-579 159 159 159 291 291 291 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 159, Max flush: 291 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.068419961Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.12499443Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.125981189Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.126038927Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.126081043Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.126100573Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"Crucible 852fcdd6-e79f-477a-84cc-ed6af361ec1c has session id: b5b1800f-4a26-45d8-95e7-5ed6c8777349","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.127424612Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Upstairs opts: Upstairs UUID: 852fcdd6-e79f-477a-84cc-ed6af361ec1c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.12747766Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Crucible stats registered with UUID: 852fcdd6-e79f-477a-84cc-ed6af361ec1c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.12749686Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:07.127512111Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.12878233Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"task":"crutest"} {"msg":"852fcdd6-e79f-477a-84cc-ed6af361ec1c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129280449Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129537835Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129651117Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129690738Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129734929Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129753477Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129800533Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129819191Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129844433Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129898704Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.12991625Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"ds_connection connected from Ok(127.0.0.1:35482)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129934808Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"ds_connection connected from Ok(127.0.0.1:50270)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129953446Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"ds_connection connected from Ok(127.0.0.1:54710)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.129971603Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.130977631Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.131061202Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.131316134Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.131731464Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.132286659Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.132377916Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133013757Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133078269Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[0]R flush_numbers[0..12]: [287, 285, 285, 285, 287, 274, 277, 290, 290, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133302658Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[0]R generation[0..12]: [158, 157, 157, 157, 158, 151, 152, 158, 158, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133339724Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133359434Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[1]R flush_numbers[0..12]: [287, 285, 285, 285, 287, 274, 277, 290, 290, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133377672Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[1]R generation[0..12]: [158, 157, 157, 157, 158, 151, 152, 158, 158, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133395077Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133412753Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[2]R flush_numbers[0..12]: [287, 285, 285, 285, 287, 274, 277, 290, 290, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.13343058Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[2]R generation[0..12]: [158, 157, 157, 157, 158, 151, 152, 158, 158, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133454188Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133480412Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133502747Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133525304Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133542078Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133558281Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"":"downstairs","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133574675Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133589575Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"0","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.13360676Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"1","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133627663Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"client":"2","":"downstairs","session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133644868Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.13366031Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Wait for a query_work_queue command to finish before sending IO 852fcdd6-e79f-477a-84cc-ed6af361ec1c is now active with session: b5b1800f-4a26-45d8-95e7-5ed6c8777349","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133687666Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133703989Z","hostname":"bmat-EVT22200007-0000589c","pid":1827,"session_id":"b5b1800f-4a26-45d8-95e7-5ed6c8777349"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133719832Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} {"msg":"Activated sub_volume 852fcdd6-e79f-477a-84cc-ed6af361ec1c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:07.133734201Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 28672 data: 13 12 15 15 17 13 14 02/30 Read block 403 len 36864 03/30 Read block 541 len 20480 04/30 Write block 461 len 4096 data: 10 05/30 Read block 546 len 12288 06/30 Write block 316 len 40960 data: 10 10 13 14 17 15 13 9 10 11 07/30 Read block 319 len 20480 08/30 Write block 88 len 40960 data: 9 9 12 10 11 11 13 15 17 18 09/30 Read block 245 len 12288 10/30 Read block 192 len 36864 11/30 Read block 371 len 12288 12/30 Read block 113 len 28672 13/30 Write block 167 len 12288 data: 18 21 18 14/30 Write block 574 len 16384 data: 11 10 11 12 15/30 Read block 565 len 32768 16/30 Read block 562 len 16384 17/30 Read block 179 len 36864 18/30 Read block 459 len 28672 19/30 Write block 503 len 16384 data: 15 16 15 17 20/30 Read block 362 len 16384 21/30 Flush 22/30 Write block 421 len 16384 data: 11 12 10 9 23/30 Read block 423 len 36864 24/30 Flush 25/30 Write block 389 len 16384 data: 14 16 20 18 26/30 Read block 480 len 40960 27/30 Write block 564 len 32768 data: 9 11 10 11 13 11 12 10 28/30 Read block 210 len 28672 29/30 Read block 433 len 12288 30/30 Read block 518 len 20480 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:07.165390298Z","hostname":"bmat-EVT22200007-0000589c","pid":1827} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 160 160 160 292 292 292 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 160 160 160 292 292 292 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 159 159 159 291 291 291 T T T 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 292 292 292 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 160 160 160 292 292 292 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 159 159 159 291 291 291 F F F 28 560-579 160 160 160 292 292 292 T T T 29 580-599 159 159 159 291 291 291 F F F Max gen: 160, Max flush: 293 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 160 160 160 292 292 292 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 156 156 156 283 283 283 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 160 160 160 292 292 292 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 161 161 161 294 294 294 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 292 292 292 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 160 160 160 292 292 292 F F F 26 520-539 154 154 154 280 280 280 F F F 27 540-559 159 159 159 291 291 291 F F F 28 560-579 161 161 161 294 294 294 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 161, Max flush: 294 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.491845485Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.545870668Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.546618828Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.546667819Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.546686617Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.546704263Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"Crucible d5fcfc77-5821-4e00-b1a4-9ed0ab34d817 has session id: 532e4e61-8dec-4c3f-83de-ae345a320103","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.548073915Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Upstairs opts: Upstairs UUID: d5fcfc77-5821-4e00-b1a4-9ed0ab34d817, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.548198039Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Crucible stats registered with UUID: d5fcfc77-5821-4e00-b1a4-9ed0ab34d817","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.548221858Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:11.548237179Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549102961Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549169287Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"d5fcfc77-5821-4e00-b1a4-9ed0ab34d817 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549196983Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549227155Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549284061Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.54932233Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549354295Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549383114Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549412825Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.54943988Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549468569Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549497278Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"ds_connection connected from Ok(127.0.0.1:36053)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549526247Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"ds_connection connected from Ok(127.0.0.1:64613)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.549555106Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"ds_connection connected from Ok(127.0.0.1:56011)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.55025596Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.550516683Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.550608852Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.551226195Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.551347373Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.551586422Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.552193764Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.552347078Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.552669488Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[0]R flush_numbers[0..12]: [287, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553508234Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[0]R generation[0..12]: [158, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553538305Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553557044Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[1]R flush_numbers[0..12]: [287, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553573948Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[1]R generation[0..12]: [158, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553590983Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553608319Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[2]R flush_numbers[0..12]: [287, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 274]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553628009Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[2]R generation[0..12]: [158, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553644773Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553661487Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.55367726Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553699635Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553715348Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553730669Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"":"downstairs","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553746421Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553760791Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"0","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553777064Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"1","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553792586Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"client":"2","":"downstairs","session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553808398Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553822928Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"d5fcfc77-5821-4e00-b1a4-9ed0ab34d817 is now active with session: 532e4e61-8dec-4c3f-83de-ae345a320103","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553838169Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553852268Z","hostname":"bmat-EVT22200007-0000589c","pid":1835,"session_id":"532e4e61-8dec-4c3f-83de-ae345a320103"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553867088Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} {"msg":"Activated sub_volume d5fcfc77-5821-4e00-b1a4-9ed0ab34d817","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:11.553881568Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 20480 data: 16 14 12 13 12 02/30 Flush 03/30 Write block 325 len 4096 data: 12 04/30 Read block 174 len 4096 05/30 Flush 06/30 Flush 07/30 Write block 239 len 36864 data: 9 9 10 12 11 12 14 14 16 08/30 Read block 428 len 8192 09/30 Read block 302 len 16384 10/30 Read block 492 len 8192 11/30 Read block 58 len 4096 12/30 Read block 61 len 32768 13/30 Read block 361 len 8192 14/30 Write block 275 len 20480 data: 12 13 12 12 15 15/30 Read block 468 len 24576 16/30 Write block 520 len 20480 data: 7 7 8 6 8 17/30 Read block 510 len 32768 18/30 Read block 430 len 32768 19/30 Read block 160 len 36864 20/30 Write block 280 len 40960 data: 14 12 11 11 9 8 7 8 7 10 21/30 Read block 103 len 8192 22/30 Read block 251 len 36864 23/30 Flush 24/30 Write block 9 len 36864 data: 14 11 9 13 12 12 10 7 7 25/30 Read block 239 len 28672 26/30 Read block 192 len 24576 27/30 Write block 389 len 40960 data: 15 17 21 19 18 17 15 15 10 10 28/30 Read block 282 len 28672 29/30 Read block 588 len 28672 30/30 Read block 335 len 20480 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:11.584610948Z","hostname":"bmat-EVT22200007-0000589c","pid":1835} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 287 287 287 T T T 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 160 160 160 292 292 292 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 162 162 162 298 298 298 F F F 13 260-279 162 162 162 298 298 298 F F F 14 280-299 162 162 162 298 298 298 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 161 161 161 294 294 294 T T T 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 292 292 292 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 160 160 160 292 292 292 F F F 26 520-539 162 162 162 298 298 298 F F F 27 540-559 162 162 162 295 295 295 F F F 28 560-579 162 162 162 295 295 295 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 162, Max flush: 298 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 299 299 299 F F F 1 020-039 157 157 157 285 285 285 F F F 2 040-059 157 157 157 285 285 285 F F F 3 060-079 157 157 157 285 285 285 F F F 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 160 160 160 292 292 292 F F F 9 180-199 154 154 154 280 280 280 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 162 162 162 298 298 298 F F F 13 260-279 162 162 162 298 298 298 F F F 14 280-299 162 162 162 298 298 298 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 F F F 19 380-399 163 163 163 299 299 299 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 160 160 160 292 292 292 F F F 23 460-479 160 160 160 292 292 292 F F F 24 480-499 154 154 154 280 280 280 F F F 25 500-519 160 160 160 292 292 292 F F F 26 520-539 162 162 162 298 298 298 F F F 27 540-559 162 162 162 295 295 295 F F F 28 560-579 162 162 162 295 295 295 F F F 29 580-599 159 159 159 291 291 291 F F F Max gen: 163, Max flush: 299 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.88726561Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.940870192Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94158245Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.941634256Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.941653245Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94167062Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"Crucible f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c has session id: eaff3f2d-2f0e-4f94-9f40-82bc8659e13d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.942627658Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Upstairs opts: Upstairs UUID: f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.942676638Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Crucible stats registered with UUID: f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.942694675Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:15.942715057Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943477687Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943511466Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943593524Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943611381Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943642064Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943658347Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943675061Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943690222Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943706656Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943721977Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943963872Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.943996859Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51537)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.944017542Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59854)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.944035228Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60901)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94425604Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.945095718Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.945300477Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.945387966Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.946218505Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.946246061Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.946285322Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947016908Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947175252Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[0]R flush_numbers[0..12]: [299, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947274726Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[0]R generation[0..12]: [163, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947297162Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94731608Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[1]R flush_numbers[0..12]: [299, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947332995Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[1]R generation[0..12]: [163, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947349298Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947365582Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[2]R flush_numbers[0..12]: [299, 285, 285, 285, 292, 274, 277, 290, 292, 280, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947388228Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[2]R generation[0..12]: [163, 157, 157, 157, 160, 151, 152, 158, 160, 154, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947404952Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947421646Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947437088Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947452219Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947467119Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947482421Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"":"downstairs","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947498253Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947512252Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"0","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947527623Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"1","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Transition from WaitQuorum to Active","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2025-03-23T06:10:15.947558136Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"client":"2","":"downstairs","session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947575892Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947590081Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c is now active with session: eaff3f2d-2f0e-4f94-9f40-82bc8659e13d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947605092Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94761907Z","hostname":"bmat-EVT22200007-0000589c","pid":1843,"session_id":"eaff3f2d-2f0e-4f94-9f40-82bc8659e13d"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.947637899Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} {"msg":"Activated sub_volume f0e9e7bd-97ba-4c9b-b9b5-0a8a58fb4d2c","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:15.94765316Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 24576 data: 17 17 17 15 14 13 02/30 Read block 140 len 16384 03/30 Read block 243 len 20480 04/30 Read block 67 len 8192 05/30 Write block 129 len 24576 data: 10 11 10 10 11 12 06/30 Read block 342 len 36864 07/30 Write block 499 len 20480 data: 11 13 15 14 16 08/30 Write block 361 len 16384 data: 13 12 14 16 09/30 Read block 386 len 28672 10/30 Write block 594 len 4096 data: 11 11/30 Read block 586 len 16384 12/30 Write block 476 len 20480 data: 10 11 12 12 13 13/30 Read block 82 len 4096 14/30 Write block 451 len 28672 data: 11 12 12 10 9 11 10 15/30 Read block 527 len 32768 16/30 Write block 533 len 36864 data: 9 11 10 12 11 13 13 12 11 17/30 Write block 260 len 20480 data: 8 10 11 11 10 18/30 Write block 284 len 20480 data: 10 9 8 9 8 19/30 Read block 413 len 28672 20/30 Write block 573 len 16384 data: 9 12 11 12 21/30 Read block 327 len 4096 22/30 Write block 178 len 24576 data: 12 13 11 11 10 8 23/30 Write block 467 len 28672 data: 12 13 14 12 11 9 8 24/30 Write block 254 len 16384 data: 14 18 18 17 25/30 Read block 393 len 24576 26/30 Read block 8 len 12288 27/30 Read block 349 len 32768 28/30 Write block 63 len 20480 data: 11 9 11 12 13 29/30 Write block 478 len 20480 data: 13 13 14 12 13 30/30 Read block 94 len 24576 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:15.976939789Z","hostname":"bmat-EVT22200007-0000589c","pid":1843} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 299 299 299 F F F 1 020-039 157 157 157 285 285 285 T T T 2 040-059 157 157 157 285 285 285 T T T 3 060-079 157 157 157 285 285 285 T T T 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 152 152 152 277 277 277 T T T 7 140-159 158 158 158 290 290 290 F F F 8 160-179 160 160 160 292 292 292 T T T 9 180-199 154 154 154 280 280 280 T T T 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 162 162 162 298 298 298 T T T 13 260-279 162 162 162 298 298 298 T T T 14 280-299 162 162 162 298 298 298 T T T 15 300-319 160 160 160 292 292 292 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 153 153 153 278 278 278 T T T 19 380-399 163 163 163 299 299 299 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 160 160 160 292 292 292 T T T 23 460-479 160 160 160 292 292 292 T T T 24 480-499 154 154 154 280 280 280 T T T 25 500-519 160 160 160 292 292 292 T T T 26 520-539 162 162 162 298 298 298 T T T 27 540-559 162 162 162 295 295 295 T T T 28 560-579 162 162 162 295 295 295 T T T 29 580-599 159 159 159 291 291 291 T T T Max gen: 163, Max flush: 299 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 299 299 299 F F F 1 020-039 164 164 164 300 300 300 F F F 2 040-059 164 164 164 300 300 300 F F F 3 060-079 164 164 164 300 300 300 F F F 4 080-099 160 160 160 292 292 292 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 164 164 164 300 300 300 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 164 164 164 300 300 300 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 164 164 164 300 300 300 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 163 163 163 299 299 299 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 164 164 164 300 300 300 F F F 23 460-479 164 164 164 300 300 300 F F F 24 480-499 164 164 164 300 300 300 F F F 25 500-519 164 164 164 300 300 300 F F F 26 520-539 164 164 164 300 300 300 F F F 27 540-559 164 164 164 300 300 300 F F F 28 560-579 164 164 164 300 300 300 F F F 29 580-599 164 164 164 300 300 300 F F F Max gen: 164, Max flush: 300 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.402409012Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.455902657Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.456651309Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.45670556Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.456724028Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.456743808Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"Crucible 2d9fe49c-a20e-4e82-8b4b-2abe83901dbf has session id: aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.457671416Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Upstairs opts: Upstairs UUID: 2d9fe49c-a20e-4e82-8b4b-2abe83901dbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.457775379Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Crucible stats registered with UUID: 2d9fe49c-a20e-4e82-8b4b-2abe83901dbf","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.457797314Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:30.457812595Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.458513229Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.458541276Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"2d9fe49c-a20e-4e82-8b4b-2abe83901dbf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.458946345Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459040537Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459423922Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459533185Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459592657Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459638731Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459684385Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.45972628Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459771172Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459815062Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36437)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459862479Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58146)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459912772Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39700)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.459965159Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.460407204Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.46046917Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.46165686Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.461692002Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.461878313Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.462637066Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.462757993Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.462904473Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[0]R flush_numbers[0..12]: [299, 300, 300, 300, 292, 274, 300, 290, 300, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.46393943Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[0]R generation[0..12]: [163, 164, 164, 164, 160, 151, 164, 158, 164, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.463976366Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.463994844Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[1]R flush_numbers[0..12]: [299, 300, 300, 300, 292, 274, 300, 290, 300, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464011828Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[1]R generation[0..12]: [163, 164, 164, 164, 160, 151, 164, 158, 164, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464029865Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464047371Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[2]R flush_numbers[0..12]: [299, 300, 300, 300, 292, 274, 300, 290, 300, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464063644Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[2]R generation[0..12]: [163, 164, 164, 164, 160, 151, 164, 158, 164, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464079617Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464096201Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Max found gen is 165","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464111222Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Generation requested: 166 >= found:165","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464131423Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464146674Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464161685Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"":"downstairs","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464177006Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464191165Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"0","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464207228Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"1","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.46422275Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"client":"2","":"downstairs","session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464238212Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464252521Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"2d9fe49c-a20e-4e82-8b4b-2abe83901dbf is now active with session: aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464267612Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.46428145Z","hostname":"bmat-EVT22200007-0000589c","pid":1851,"session_id":"aedb8b0c-eecc-4a1c-b068-a15fa4bc42d5"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:30.464295619Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} {"msg":"Activated sub_volume 2d9fe49c-a20e-4e82-8b4b-2abe83901dbf","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-23T06:10:30.464308746Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 107 len 32768 data: 12 14 14 16 14 14 17 17 02/30 Write block 84 len 28672 data: 14 14 11 10 10 10 13 03/30 Read block 377 len 24576 04/30 Flush 05/30 Write block 256 len 4096 data: 19 06/30 Read block 404 len 16384 07/30 Write block 325 len 36864 data: 13 9 9 11 13 16 16 16 17 08/30 Read block 99 len 8192 09/30 Read block 411 len 32768 10/30 Read block 337 len 12288 11/30 Read block 227 len 40960 12/30 Flush 13/30 Write block 447 len 40960 data: 16 18 14 15 12 13 13 11 10 12 14/30 Read block 351 len 8192 15/30 Read block 595 len 20480 16/30 Read block 17 len 4096 17/30 Write block 507 len 24576 data: 17 17 15 17 16 14 18/30 Write block 584 len 40960 data: 15 14 16 15 15 15 11 12 11 8 19/30 Read block 506 len 28672 20/30 Read block 127 len 36864 21/30 Read block 269 len 28672 22/30 Write block 557 len 20480 data: 18 17 15 13 14 23/30 Read block 428 len 24576 24/30 Write block 56 len 40960 data: 13 17 15 14 17 15 13 12 10 12 25/30 Write block 166 len 28672 data: 12 19 22 19 18 18 16 26/30 Read block 135 len 8192 27/30 Read block 593 len 16384 28/30 Read block 47 len 4096 29/30 Read block 349 len 36864 30/30 Read block 445 len 16384 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:30.494104426Z","hostname":"bmat-EVT22200007-0000589c","pid":1851} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 299 299 299 F F F 1 020-039 164 164 164 300 300 300 F F F 2 040-059 164 164 164 300 300 300 T T T 3 060-079 164 164 164 300 300 300 T T T 4 080-099 166 166 166 301 301 301 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 300 300 300 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 164 164 164 300 300 300 T T T 9 180-199 164 164 164 300 300 300 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 166 166 166 302 302 302 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 163 163 163 299 299 299 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 164 164 164 300 300 300 T T T 23 460-479 164 164 164 300 300 300 F F F 24 480-499 164 164 164 300 300 300 F F F 25 500-519 164 164 164 300 300 300 T T T 26 520-539 164 164 164 300 300 300 F F F 27 540-559 164 164 164 300 300 300 T T T 28 560-579 164 164 164 300 300 300 T T T 29 580-599 164 164 164 300 300 300 T T T Max gen: 166, Max flush: 302 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 299 299 299 F F F 1 020-039 164 164 164 300 300 300 F F F 2 040-059 167 167 167 303 303 303 F F F 3 060-079 167 167 167 303 303 303 F F F 4 080-099 166 166 166 301 301 301 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 300 300 300 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 164 164 164 300 300 300 F F F 15 300-319 160 160 160 292 292 292 F F F 16 320-339 166 166 166 302 302 302 F F F 17 340-359 156 156 156 284 284 284 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 163 163 163 299 299 299 F F F 20 400-419 158 158 158 286 286 286 F F F 21 420-439 160 160 160 293 293 293 F F F 22 440-459 167 167 167 303 303 303 F F F 23 460-479 164 164 164 300 300 300 F F F 24 480-499 164 164 164 300 300 300 F F F 25 500-519 167 167 167 303 303 303 F F F 26 520-539 164 164 164 300 300 300 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 167 167 167 303 303 303 F F F 29 580-599 167 167 167 303 303 303 F F F Max gen: 167, Max flush: 303 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.844234612Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:34.896492629Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.89689855Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.934486463Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.936246103Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.936325826Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.936356058Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.936395589Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"Crucible 44496ad6-a25a-4b1c-8b51-350a09a638cd has session id: 888d8ed9-c2cc-4b2a-89ce-474d4dda06fd","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.937705739Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Upstairs opts: Upstairs UUID: 44496ad6-a25a-4b1c-8b51-350a09a638cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.937781003Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Crucible stats registered with UUID: 44496ad6-a25a-4b1c-8b51-350a09a638cd","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.93780388Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:34.937821807Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938568244Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938601241Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"44496ad6-a25a-4b1c-8b51-350a09a638cd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938815179Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938839348Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938871875Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938891104Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938910674Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938928982Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.938948872Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.939051793Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.939082235Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.939101805Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47180)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.939523328Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60805)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.93960769Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:39772)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.939627902Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.940426415Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.940464574Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.940482711Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.941213406Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.941383474Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.941408164Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942006137Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.94214897Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[0]R flush_numbers[0..12]: [299, 300, 303, 303, 301, 301, 300, 290, 303, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942317134Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[0]R generation[0..12]: [163, 164, 167, 167, 166, 166, 164, 158, 167, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942342005Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942367417Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[1]R flush_numbers[0..12]: [299, 300, 303, 303, 301, 301, 300, 290, 303, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942385674Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[1]R generation[0..12]: [163, 164, 167, 167, 166, 166, 164, 158, 167, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942402919Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942423251Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[2]R flush_numbers[0..12]: [299, 300, 303, 303, 301, 301, 300, 290, 303, 300, 282, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942440386Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[2]R generation[0..12]: [163, 164, 167, 167, 166, 166, 164, 158, 167, 164, 156, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942457591Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942475017Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942490649Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942506541Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942522314Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942538196Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"":"downstairs","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.94255457Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942569781Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"0","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942586745Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"1","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:10:34.942607307Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"client":"2","":"downstairs","session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.94263776Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942652831Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"44496ad6-a25a-4b1c-8b51-350a09a638cd is now active with session: 888d8ed9-c2cc-4b2a-89ce-474d4dda06fd","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942668583Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942683123Z","hostname":"bmat-EVT22200007-0000589c","pid":1859,"session_id":"888d8ed9-c2cc-4b2a-89ce-474d4dda06fd"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942698223Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} {"msg":"Activated sub_volume 44496ad6-a25a-4b1c-8b51-350a09a638cd","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:34.942712092Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 20480 data: 8 10 13 9 9 02/30 Flush 03/30 Write block 365 len 36864 data: 20 20 14 15 11 11 13 15 16 04/30 Flush 05/30 Write block 397 len 28672 data: 11 11 11 13 16 13 15 06/30 Read block 493 len 32768 07/30 Write block 291 len 40960 data: 11 10 12 16 17 18 19 15 12 13 08/30 Read block 469 len 28672 09/30 Read block 7 len 8192 10/30 Flush 11/30 Write block 16 len 36864 data: 8 8 7 5 5 4 10 10 11 12/30 Write block 530 len 36864 data: 13 13 10 10 12 11 13 12 14 13/30 Read block 304 len 24576 14/30 Write block 401 len 4096 data: 17 15/30 Write block 41 len 36864 data: 18 16 15 14 13 13 14 10 12 16/30 Read block 343 len 16384 17/30 Flush 18/30 Write block 591 len 8192 data: 13 12 19/30 Write block 231 len 4096 data: 13 20/30 Write block 533 len 24576 data: 11 13 12 14 13 15 21/30 Write block 512 len 16384 data: 15 13 13 13 22/30 Write block 422 len 40960 data: 13 11 10 8 6 6 8 11 12 13 23/30 Read block 126 len 16384 24/30 Read block 193 len 16384 25/30 Write block 466 len 32768 data: 13 13 14 15 13 12 10 9 26/30 Read block 51 len 28672 27/30 Read block 327 len 32768 28/30 Write block 248 len 28672 data: 15 16 16 17 14 15 15 29/30 Write block 334 len 36864 data: 16 13 13 14 13 14 14 15 19 30/30 Read block 408 len 12288 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 13 11 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:34.972402938Z","hostname":"bmat-EVT22200007-0000589c","pid":1859} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 307 307 307 F F F 1 020-039 168 168 168 307 307 307 F F F 2 040-059 168 168 168 307 307 307 F F F 3 060-079 167 167 167 303 303 303 F F F 4 080-099 166 166 166 301 301 301 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 300 300 300 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 162 162 162 298 298 298 T T T 12 240-259 166 166 166 302 302 302 T T T 13 260-279 164 164 164 300 300 300 F F F 14 280-299 168 168 168 306 306 306 F F F 15 300-319 168 168 168 306 306 306 F F F 16 320-339 166 166 166 302 302 302 T T T 17 340-359 156 156 156 284 284 284 T T T 18 360-379 168 168 168 305 305 305 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 160 160 160 293 293 293 T T T 22 440-459 167 167 167 303 303 303 F F F 23 460-479 164 164 164 300 300 300 T T T 24 480-499 168 168 168 304 304 304 F F F 25 500-519 167 167 167 303 303 303 T T T 26 520-539 168 168 168 307 307 307 T T T 27 540-559 167 167 167 303 303 303 F F F 28 560-579 167 167 167 303 303 303 F F F 29 580-599 167 167 167 303 303 303 T T T Max gen: 168, Max flush: 307 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 307 307 307 F F F 1 020-039 168 168 168 307 307 307 F F F 2 040-059 168 168 168 307 307 307 F F F 3 060-079 167 167 167 303 303 303 F F F 4 080-099 166 166 166 301 301 301 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 300 300 300 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 156 156 156 282 282 282 F F F 11 220-239 169 169 169 308 308 308 F F F 12 240-259 169 169 169 308 308 308 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 168 168 168 306 306 306 F F F 15 300-319 168 168 168 306 306 306 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 168 168 168 305 305 305 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 169 169 169 308 308 308 F F F 22 440-459 167 167 167 303 303 303 F F F 23 460-479 169 169 169 308 308 308 F F F 24 480-499 168 168 168 304 304 304 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 169 169 169 308 308 308 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 167 167 167 303 303 303 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 169, Max flush: 308 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.360280212Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.414701816Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.416299896Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.416354608Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.41637509Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.416393247Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"Crucible 6d234e98-b8a7-42dd-b8d5-5b12b3802460 has session id: f2ac337d-4151-46ce-b922-d2b7c5dfc41e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.417468958Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Upstairs opts: Upstairs UUID: 6d234e98-b8a7-42dd-b8d5-5b12b3802460, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.41754816Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Crucible stats registered with UUID: 6d234e98-b8a7-42dd-b8d5-5b12b3802460","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.417569063Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:39.417585727Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418326332Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418649102Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"6d234e98-b8a7-42dd-b8d5-5b12b3802460 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.41866738Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418690858Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418722042Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418739297Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418756502Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418772154Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.418789019Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419207164Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419231514Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419249871Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61349)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419267047Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58656)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419335076Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62391)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.419356029Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.420269808Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.420315722Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.42033416Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.421248601Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.421278823Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.421392245Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422258317Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422317357Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[0]R flush_numbers[0..12]: [307, 307, 307, 303, 301, 301, 300, 290, 303, 300, 282, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422351497Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[0]R generation[0..12]: [168, 168, 168, 167, 166, 166, 164, 158, 167, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422371638Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422389735Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[1]R flush_numbers[0..12]: [307, 307, 307, 303, 301, 301, 300, 290, 303, 300, 282, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422406951Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[1]R generation[0..12]: [168, 168, 168, 167, 166, 166, 164, 158, 167, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422423595Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422440319Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[2]R flush_numbers[0..12]: [307, 307, 307, 303, 301, 301, 300, 290, 303, 300, 282, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422456793Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[2]R generation[0..12]: [168, 168, 168, 167, 166, 166, 164, 158, 167, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422473106Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.4224899Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422511154Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422527948Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.4225435Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422558771Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"":"downstairs","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422574483Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422589103Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"0","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422606018Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"1","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422622071Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"client":"2","":"downstairs","session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422638023Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422652533Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"6d234e98-b8a7-42dd-b8d5-5b12b3802460 is now active with session: f2ac337d-4151-46ce-b922-d2b7c5dfc41e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422667925Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422681913Z","hostname":"bmat-EVT22200007-0000589c","pid":1867,"session_id":"f2ac337d-4151-46ce-b922-d2b7c5dfc41e"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422696563Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} {"msg":"Activated sub_volume 6d234e98-b8a7-42dd-b8d5-5b12b3802460","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:39.422710111Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 520 len 24576 data: 8 8 9 7 9 9 03/30 Write block 239 len 16384 data: 10 10 11 13 04/30 Flush 05/30 Write block 461 len 36864 data: 11 11 13 14 15 14 14 15 16 06/30 Read block 34 len 4096 07/30 Read block 242 len 4096 08/30 Write block 116 len 36864 data: 18 15 14 10 11 7 7 6 10 09/30 Read block 456 len 16384 10/30 Write block 427 len 12288 data: 7 9 12 11/30 Flush 12/30 Write block 435 len 8192 data: 14 12 13/30 Read block 448 len 12288 14/30 Write block 124 len 24576 data: 11 11 12 11 11 11 15/30 Flush 16/30 Write block 425 len 8192 data: 9 7 17/30 Read block 572 len 16384 18/30 Write block 215 len 4096 data: 4 19/30 Read block 83 len 36864 20/30 Read block 370 len 28672 21/30 Read block 357 len 32768 22/30 Read block 194 len 40960 23/30 Read block 209 len 36864 24/30 Flush 25/30 Flush 26/30 Write block 217 len 32768 data: 11 11 11 13 12 16 17 16 27/30 Write block 8 len 36864 data: 15 15 12 10 14 13 13 11 9 28/30 Read block 204 len 12288 29/30 Write block 66 len 20480 data: 13 14 12 10 9 30/30 Read block 537 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:39.451244411Z","hostname":"bmat-EVT22200007-0000589c","pid":1867} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 307 307 307 T T T 1 020-039 168 168 168 307 307 307 F F F 2 040-059 168 168 168 307 307 307 F F F 3 060-079 167 167 167 303 303 303 T T T 4 080-099 166 166 166 301 301 301 F F F 5 100-119 170 170 170 311 311 311 F F F 6 120-139 170 170 170 312 312 312 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 170 170 170 313 313 313 T T T 11 220-239 170 170 170 310 310 310 T T T 12 240-259 170 170 170 310 310 310 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 168 168 168 306 306 306 F F F 15 300-319 168 168 168 306 306 306 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 168 168 168 305 305 305 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 170 170 170 313 313 313 F F F 22 440-459 167 167 167 303 303 303 F F F 23 460-479 170 170 170 311 311 311 F F F 24 480-499 168 168 168 304 304 304 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 167 167 167 303 303 303 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 170, Max flush: 313 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 314 314 314 F F F 1 020-039 168 168 168 307 307 307 F F F 2 040-059 168 168 168 307 307 307 F F F 3 060-079 171 171 171 314 314 314 F F F 4 080-099 166 166 166 301 301 301 F F F 5 100-119 170 170 170 311 311 311 F F F 6 120-139 170 170 170 312 312 312 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 171 171 171 314 314 314 F F F 12 240-259 170 170 170 310 310 310 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 168 168 168 306 306 306 F F F 15 300-319 168 168 168 306 306 306 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 168 168 168 305 305 305 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 170 170 170 313 313 313 F F F 22 440-459 167 167 167 303 303 303 F F F 23 460-479 170 170 170 311 311 311 F F F 24 480-499 168 168 168 304 304 304 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 167 167 167 303 303 303 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 171, Max flush: 314 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.771748753Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.829837279Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.830748554Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.830867387Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.830898821Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.830929464Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"Crucible 12a77017-318b-46ed-a13d-9e880bee96e2 has session id: d3d46924-10fd-44e0-92d0-34ee4d3bb774","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.831876241Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Upstairs opts: Upstairs UUID: 12a77017-318b-46ed-a13d-9e880bee96e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83195886Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Crucible stats registered with UUID: 12a77017-318b-46ed-a13d-9e880bee96e2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.831987328Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:43.832003371Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832697822Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832732353Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"12a77017-318b-46ed-a13d-9e880bee96e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832854693Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83287761Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832907581Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832924636Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832944677Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832961692Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832979689Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.832995862Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83308772Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83311228Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"ds_connection connected from Ok(127.0.0.1:37347)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.833549675Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"ds_connection connected from Ok(127.0.0.1:42691)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.833624167Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"ds_connection connected from Ok(127.0.0.1:46312)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.833647074Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.834335372Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.834542807Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.834567477Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.835269914Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.835326861Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.835534866Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83590315Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836129132Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[0]R flush_numbers[0..12]: [314, 307, 307, 314, 301, 311, 312, 290, 303, 300, 314, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836239268Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[0]R generation[0..12]: [171, 168, 168, 171, 166, 170, 170, 158, 167, 164, 171, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836259529Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836277335Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[1]R flush_numbers[0..12]: [314, 307, 307, 314, 301, 311, 312, 290, 303, 300, 314, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836293789Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[1]R generation[0..12]: [171, 168, 168, 171, 166, 170, 170, 158, 167, 164, 171, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836319281Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836336577Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[2]R flush_numbers[0..12]: [314, 307, 307, 314, 301, 311, 312, 290, 303, 300, 314, 314]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836353431Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[2]R generation[0..12]: [171, 168, 168, 171, 166, 170, 170, 158, 167, 164, 171, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836369815Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836388242Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836503598Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836598202Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836618513Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836633875Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"":"downstairs","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:10:43.836649326Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836684037Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"0","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.83670016Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"1","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836715792Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"client":"2","":"downstairs","session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836732316Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836746716Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"12a77017-318b-46ed-a13d-9e880bee96e2 is now active with session: d3d46924-10fd-44e0-92d0-34ee4d3bb774","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836828403Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836907745Z","hostname":"bmat-EVT22200007-0000589c","pid":1875,"session_id":"d3d46924-10fd-44e0-92d0-34ee4d3bb774"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836928297Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} {"msg":"Activated sub_volume 12a77017-318b-46ed-a13d-9e880bee96e2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:43.836942246Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 16384 data: 10 9 10 14 02/30 Write block 552 len 24576 data: 16 16 18 15 17 19 03/30 Read block 73 len 24576 04/30 Read block 545 len 8192 05/30 Read block 307 len 32768 06/30 Write block 263 len 36864 data: 12 11 12 13 11 12 12 12 11 07/30 Read block 125 len 8192 08/30 Read block 124 len 16384 09/30 Read block 292 len 24576 10/30 Read block 4 len 40960 11/30 Write block 38 len 40960 data: 20 18 18 19 17 16 15 14 14 15 12/30 Write block 448 len 4096 data: 19 13/30 Write block 353 len 40960 data: 11 11 13 14 16 16 17 14 14 13 14/30 Write block 11 len 4096 data: 11 15/30 Write block 90 len 16384 data: 14 11 12 12 16/30 Read block 10 len 12288 17/30 Read block 229 len 36864 18/30 Read block 151 len 16384 19/30 Write block 493 len 24576 data: 14 10 10 9 11 10 20/30 Write block 560 len 36864 data: 14 15 13 9 10 12 11 12 14 21/30 Write block 237 len 20480 data: 14 14 11 11 12 22/30 Write block 354 len 16384 data: 12 14 15 17 23/30 Write block 442 len 20480 data: 12 11 14 13 16 24/30 Read block 200 len 20480 25/30 Read block 185 len 36864 26/30 Read block 202 len 40960 27/30 Write block 310 len 16384 data: 10 10 7 7 28/30 Write block 181 len 28672 data: 12 11 9 10 12 14 14 29/30 Read block 517 len 40960 30/30 Read block 423 len 16384 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:43.86276586Z","hostname":"bmat-EVT22200007-0000589c","pid":1875} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 314 314 314 T T T 1 020-039 168 168 168 307 307 307 T T T 2 040-059 168 168 168 307 307 307 T T T 3 060-079 171 171 171 314 314 314 F F F 4 080-099 166 166 166 301 301 301 T T T 5 100-119 170 170 170 311 311 311 F F F 6 120-139 170 170 170 312 312 312 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 164 164 164 300 300 300 T T T 10 200-219 171 171 171 314 314 314 F F F 11 220-239 171 171 171 314 314 314 T T T 12 240-259 170 170 170 310 310 310 T T T 13 260-279 164 164 164 300 300 300 T T T 14 280-299 168 168 168 306 306 306 F F F 15 300-319 168 168 168 306 306 306 T T T 16 320-339 169 169 169 308 308 308 F F F 17 340-359 169 169 169 308 308 308 T T T 18 360-379 168 168 168 305 305 305 T T T 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 170 170 170 313 313 313 F F F 22 440-459 167 167 167 303 303 303 T T T 23 460-479 170 170 170 311 311 311 F F F 24 480-499 168 168 168 304 304 304 T T T 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 167 167 167 303 303 303 T T T 28 560-579 167 167 167 303 303 303 T T T 29 580-599 169 169 169 308 308 308 F F F Max gen: 171, Max flush: 314 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 315 315 315 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 172 172 172 315 315 315 F F F 3 060-079 171 171 171 314 314 314 F F F 4 080-099 172 172 172 315 315 315 F F F 5 100-119 170 170 170 311 311 311 F F F 6 120-139 170 170 170 312 312 312 F F F 7 140-159 158 158 158 290 290 290 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 172 172 172 315 315 315 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 172 172 172 315 315 315 F F F 13 260-279 172 172 172 315 315 315 F F F 14 280-299 168 168 168 306 306 306 F F F 15 300-319 172 172 172 315 315 315 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 172 172 172 315 315 315 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 170 170 170 313 313 313 F F F 22 440-459 172 172 172 315 315 315 F F F 23 460-479 170 170 170 311 311 311 F F F 24 480-499 172 172 172 315 315 315 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 172 172 172 315 315 315 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 172, Max flush: 315 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.258325842Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:48.309509832Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.309941956Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.346916664Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.347688203Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.347761733Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.34777955Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.347798218Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"Crucible 181131d4-76f4-4dff-adfc-2f42e31018d2 has session id: fbbce9be-6c78-467b-8062-6bf35724ebec","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.348696997Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Upstairs opts: Upstairs UUID: 181131d4-76f4-4dff-adfc-2f42e31018d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.348748071Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Crucible stats registered with UUID: 181131d4-76f4-4dff-adfc-2f42e31018d2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.348767621Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:48.348782752Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349573199Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349610656Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"181131d4-76f4-4dff-adfc-2f42e31018d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349723206Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349744159Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.34977398Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349791275Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349808881Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349824824Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349841819Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349858002Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349875358Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.349898365Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63116)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.350461066Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"ds_connection connected from Ok(127.0.0.1:41749)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.350533364Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63861)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.350555008Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.351048868Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.351534983Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.351561457Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.352309267Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.352595633Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.352783337Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353142422Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353428848Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[0]R flush_numbers[0..12]: [315, 315, 315, 314, 315, 311, 312, 290, 303, 315, 314, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.35347393Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[0]R generation[0..12]: [172, 172, 172, 171, 172, 170, 170, 158, 167, 172, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353497669Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353516166Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[1]R flush_numbers[0..12]: [315, 315, 315, 314, 315, 311, 312, 290, 303, 315, 314, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353533472Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[1]R generation[0..12]: [172, 172, 172, 171, 172, 170, 170, 158, 167, 172, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353551288Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353568724Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[2]R flush_numbers[0..12]: [315, 315, 315, 314, 315, 311, 312, 290, 303, 315, 314, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353586059Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[2]R generation[0..12]: [172, 172, 172, 171, 172, 170, 170, 158, 167, 172, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353611351Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353630961Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Max found gen is 173","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353646864Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Generation requested: 174 >= found:173","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353663077Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353678739Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353694381Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"":"downstairs","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353710264Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353725094Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"0","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353743311Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"1","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353759304Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"client":"2","":"downstairs","session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353777421Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353792442Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"181131d4-76f4-4dff-adfc-2f42e31018d2 is now active with session: fbbce9be-6c78-467b-8062-6bf35724ebec","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353807953Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353822373Z","hostname":"bmat-EVT22200007-0000589c","pid":1883,"session_id":"fbbce9be-6c78-467b-8062-6bf35724ebec"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353837394Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} {"msg":"Activated sub_volume 181131d4-76f4-4dff-adfc-2f42e31018d2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:48.353850681Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 40960 data: 16 16 12 15 18 18 17 15 13 13 02/30 Flush 03/30 Write block 159 len 4096 data: 13 04/30 Write block 494 len 8192 data: 11 11 05/30 Write block 284 len 16384 data: 11 10 9 10 06/30 Write block 12 len 40960 data: 15 14 14 12 10 9 8 6 6 5 07/30 Write block 407 len 40960 data: 14 18 18 17 16 14 12 10 8 6 08/30 Write block 404 len 20480 data: 18 19 16 15 19 09/30 Flush 10/30 Write block 106 len 32768 data: 13 13 15 15 17 15 15 18 11/30 Read block 270 len 28672 12/30 Read block 105 len 32768 13/30 Write block 544 len 20480 data: 11 7 11 13 14 14/30 Read block 155 len 4096 15/30 Write block 429 len 32768 data: 13 13 14 16 17 14 15 13 16/30 Write block 39 len 24576 data: 19 19 20 18 17 16 17/30 Write block 292 len 12288 data: 11 13 17 18/30 Read block 55 len 20480 19/30 Write block 150 len 16384 data: 14 14 10 8 20/30 Write block 112 len 36864 data: 16 19 18 17 19 16 15 11 12 21/30 Write block 99 len 28672 data: 14 13 10 13 12 13 14 22/30 Write block 264 len 24576 data: 12 13 14 12 13 13 23/30 Read block 172 len 20480 24/30 Read block 91 len 40960 25/30 Read block 38 len 24576 26/30 Read block 518 len 8192 27/30 Read block 555 len 40960 28/30 Read block 379 len 16384 29/30 Read block 586 len 12288 30/30 Write block 178 len 40960 data: 13 14 12 13 12 10 11 13 15 15 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 8 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:48.381674447Z","hostname":"bmat-EVT22200007-0000589c","pid":1883} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 317 317 317 F F F 1 020-039 174 174 174 317 317 317 T T T 2 040-059 172 172 172 315 315 315 T T T 3 060-079 171 171 171 314 314 314 F F F 4 080-099 172 172 172 315 315 315 T T T 5 100-119 170 170 170 311 311 311 T T T 6 120-139 170 170 170 312 312 312 T T T 7 140-159 174 174 174 317 317 317 T T T 8 160-179 167 167 167 303 303 303 T T T 9 180-199 174 174 174 316 316 316 T T T 10 200-219 174 174 174 316 316 316 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 172 172 172 315 315 315 F F F 13 260-279 172 172 172 315 315 315 T T T 14 280-299 174 174 174 317 317 317 T T T 15 300-319 172 172 172 315 315 315 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 172 172 172 315 315 315 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 174 174 174 317 317 317 F F F 21 420-439 170 170 170 313 313 313 T T T 22 440-459 172 172 172 315 315 315 F F F 23 460-479 170 170 170 311 311 311 F F F 24 480-499 174 174 174 317 317 317 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 172 172 172 315 315 315 T T T 28 560-579 172 172 172 315 315 315 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 174, Max flush: 317 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 317 317 317 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 175 175 175 318 318 318 F F F 3 060-079 171 171 171 314 314 314 F F F 4 080-099 175 175 175 318 318 318 F F F 5 100-119 175 175 175 318 318 318 F F F 6 120-139 175 175 175 318 318 318 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 175 175 175 318 318 318 F F F 9 180-199 175 175 175 318 318 318 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 172 172 172 315 315 315 F F F 13 260-279 175 175 175 318 318 318 F F F 14 280-299 175 175 175 318 318 318 F F F 15 300-319 172 172 172 315 315 315 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 172 172 172 315 315 315 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 174 174 174 317 317 317 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 172 172 172 315 315 315 F F F 23 460-479 170 170 170 311 311 311 F F F 24 480-499 174 174 174 317 317 317 F F F 25 500-519 169 169 169 308 308 308 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 175 175 175 318 318 318 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 175, Max flush: 318 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.787687525Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:52.839603333Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.840038142Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.876848795Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.877547865Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.877594821Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.877613289Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.877631536Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"Crucible f44cc4ae-4d58-491f-af21-d4b374d2c94e has session id: e6e86d48-4f00-46c9-aafa-35c2726409a6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.878591651Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Upstairs opts: Upstairs UUID: f44cc4ae-4d58-491f-af21-d4b374d2c94e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.878639078Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Crucible stats registered with UUID: f44cc4ae-4d58-491f-af21-d4b374d2c94e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.878657325Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:52.878672536Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879472473Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879508356Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"f44cc4ae-4d58-491f-af21-d4b374d2c94e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879640156Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879665137Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879889757Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879944178Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.87996972Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.879986735Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.880004792Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.880021356Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.880038461Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.880055666Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48038)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.88056089Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55628)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.880624159Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47162)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.8806442Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.881339493Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.88137245Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.881390778Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.882171234Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.882330861Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.882381435Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883080284Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.88311258Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[0]R flush_numbers[0..12]: [317, 318, 318, 314, 318, 318, 318, 318, 318, 318, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883271235Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[0]R generation[0..12]: [174, 175, 175, 171, 175, 175, 175, 175, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883295144Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883313351Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[1]R flush_numbers[0..12]: [317, 318, 318, 314, 318, 318, 318, 318, 318, 318, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883331228Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[1]R generation[0..12]: [174, 175, 175, 171, 175, 175, 175, 175, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883348373Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883367742Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[2]R flush_numbers[0..12]: [317, 318, 318, 314, 318, 318, 318, 318, 318, 318, 316, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883385208Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[2]R generation[0..12]: [174, 175, 175, 171, 175, 175, 175, 175, 175, 175, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883401762Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883418546Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883434138Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883449991Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883471465Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883487467Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"":"downstairs","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883503109Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883517689Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"0","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883533572Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"1","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883549244Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"client":"2","":"downstairs","session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883565206Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883579706Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"f44cc4ae-4d58-491f-af21-d4b374d2c94e is now active with session: e6e86d48-4f00-46c9-aafa-35c2726409a6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883595278Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883609527Z","hostname":"bmat-EVT22200007-0000589c","pid":1891,"session_id":"e6e86d48-4f00-46c9-aafa-35c2726409a6"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883624177Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} {"msg":"Activated sub_volume f44cc4ae-4d58-491f-af21-d4b374d2c94e","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:52.883637865Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 36864 data: 16 11 13 13 12 17 19 18 16 02/30 Write block 509 len 12288 data: 16 18 17 03/30 Read block 92 len 4096 04/30 Write block 457 len 8192 data: 11 11 05/30 Write block 411 len 32768 data: 17 15 13 11 9 7 8 8 06/30 Write block 340 len 32768 data: 15 16 20 15 18 17 15 16 07/30 Write block 58 len 28672 data: 16 15 18 16 14 13 11 08/30 Write block 104 len 40960 data: 14 15 14 14 16 16 18 16 17 20 09/30 Read block 444 len 20480 10/30 Read block 178 len 4096 11/30 Read block 199 len 4096 12/30 Read block 249 len 12288 13/30 Read block 153 len 32768 14/30 Write block 206 len 24576 data: 14 11 12 11 12 7 15/30 Read block 485 len 24576 16/30 Read block 261 len 8192 17/30 Write block 260 len 28672 data: 9 11 12 13 13 14 15 18/30 Write block 88 len 4096 data: 11 19/30 Write block 122 len 24576 data: 8 7 12 12 13 12 20/30 Write block 512 len 24576 data: 16 14 14 14 13 13 21/30 Read block 452 len 4096 22/30 Write block 203 len 8192 data: 15 16 23/30 Read block 69 len 32768 24/30 Read block 567 len 4096 25/30 Write block 297 len 20480 data: 20 16 13 14 11 26/30 Write block 96 len 12288 data: 18 19 16 27/30 Read block 463 len 28672 28/30 Read block 204 len 28672 29/30 Write block 467 len 28672 data: 15 16 17 14 13 11 10 30/30 Read block 509 len 4096 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 30 30 28 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:52.910242447Z","hostname":"bmat-EVT22200007-0000589c","pid":1891} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 317 317 317 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 175 175 175 318 318 318 T T T 3 060-079 171 171 171 314 314 314 T T T 4 080-099 175 175 175 318 318 318 T T T 5 100-119 175 175 175 318 318 318 T T T 6 120-139 175 175 175 318 318 318 T T T 7 140-159 175 175 175 318 318 318 F F F 8 160-179 175 175 175 318 318 318 F F F 9 180-199 175 175 175 318 318 318 F F F 10 200-219 174 174 174 316 316 316 T T T 11 220-239 172 172 172 315 315 315 F F F 12 240-259 172 172 172 315 315 315 F F F 13 260-279 175 175 175 318 318 318 T T T 14 280-299 175 175 175 318 318 318 T T T 15 300-319 172 172 172 315 315 315 T T T 16 320-339 169 169 169 308 308 308 F F F 17 340-359 172 172 172 315 315 315 T T T 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 174 174 174 317 317 317 T T T 21 420-439 175 175 175 318 318 318 F F F 22 440-459 172 172 172 315 315 315 T T T 23 460-479 170 170 170 311 311 311 T T T 24 480-499 174 174 174 317 317 317 F F F 25 500-519 169 169 169 308 308 308 T T T 26 520-539 170 170 170 310 310 310 F F F 27 540-559 175 175 175 318 318 318 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 175, Max flush: 318 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 317 317 317 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 176 176 176 319 319 319 F F F 3 060-079 176 176 176 319 319 319 F F F 4 080-099 176 176 176 319 319 319 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 176 176 176 319 319 319 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 175 175 175 318 318 318 F F F 9 180-199 175 175 175 318 318 318 F F F 10 200-219 176 176 176 319 319 319 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 172 172 172 315 315 315 F F F 13 260-279 176 176 176 319 319 319 F F F 14 280-299 176 176 176 319 319 319 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 176 176 176 319 319 319 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 176 176 176 319 319 319 F F F 23 460-479 176 176 176 319 319 319 F F F 24 480-499 174 174 174 317 317 317 F F F 25 500-519 176 176 176 319 319 319 F F F 26 520-539 170 170 170 310 310 310 F F F 27 540-559 175 175 175 318 318 318 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 169 169 169 308 308 308 F F F Max gen: 176, Max flush: 319 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.338128257Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.391959132Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.392742264Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39279889Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.392819061Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.392837599Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"Crucible 478f84a9-9c98-459b-8a08-e784f0301326 has session id: 27bbd241-02d3-4b27-a807-f1e2e216aa72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.393609829Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Upstairs opts: Upstairs UUID: 478f84a9-9c98-459b-8a08-e784f0301326, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.393762191Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Crucible stats registered with UUID: 478f84a9-9c98-459b-8a08-e784f0301326","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.393786531Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:57.393802463Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394415708Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394445128Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"478f84a9-9c98-459b-8a08-e784f0301326 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39455368Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394573821Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394605777Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394623172Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.3946416Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394657893Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39467595Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394691903Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39471485Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394745052Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"ds_connection connected from Ok(127.0.0.1:47688)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.394870248Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"ds_connection connected from Ok(127.0.0.1:56513)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.395233852Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"ds_connection connected from Ok(127.0.0.1:46536)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.395269695Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.395644211Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.395805712Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39635694Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.396492647Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39709513Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39720747Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.398172244Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.398829027Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[0]R flush_numbers[0..12]: [317, 318, 319, 319, 319, 319, 319, 318, 318, 318, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399483757Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[0]R generation[0..12]: [174, 175, 176, 176, 176, 176, 176, 175, 175, 175, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399510341Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399535813Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[1]R flush_numbers[0..12]: [317, 318, 319, 319, 319, 319, 319, 318, 318, 318, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39955369Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[1]R generation[0..12]: [174, 175, 176, 176, 176, 176, 176, 175, 175, 175, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399570775Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399588461Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[2]R flush_numbers[0..12]: [317, 318, 319, 319, 319, 319, 319, 318, 318, 318, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399605566Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[2]R generation[0..12]: [174, 175, 176, 176, 176, 176, 176, 175, 175, 175, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39962196Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399639045Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Max found gen is 177","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399654727Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Generation requested: 178 >= found:177","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399670409Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39968584Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399701092Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"":"downstairs","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399716934Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399731093Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"0","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399747106Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"1","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399766355Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"client":"2","":"downstairs","session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399783099Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399797789Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"478f84a9-9c98-459b-8a08-e784f0301326 is now active with session: 27bbd241-02d3-4b27-a807-f1e2e216aa72","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399813171Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39982743Z","hostname":"bmat-EVT22200007-0000589c","pid":1899,"session_id":"27bbd241-02d3-4b27-a807-f1e2e216aa72"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.399842771Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} {"msg":"Activated sub_volume 478f84a9-9c98-459b-8a08-e784f0301326","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:10:57.39985679Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 280 len 28672 data: 15 13 12 12 12 11 10 02/30 Write block 552 len 28672 data: 17 17 19 16 18 20 18 03/30 Write block 517 len 40960 data: 14 11 8 9 9 10 8 10 10 11 04/30 Read block 157 len 16384 05/30 Write block 184 len 12288 data: 12 14 16 06/30 Flush 07/30 Write block 171 len 28672 data: 19 17 17 17 16 14 11 08/30 Write block 83 len 36864 data: 14 15 15 12 11 12 11 15 12 09/30 Read block 216 len 24576 10/30 Read block 518 len 28672 11/30 Write block 414 len 24576 data: 12 10 8 9 9 10 12/30 Read block 201 len 24576 13/30 Read block 400 len 32768 14/30 Write block 13 len 4096 data: 15 15/30 Write block 509 len 36864 data: 17 19 18 17 15 15 15 14 15 16/30 Read block 338 len 8192 17/30 Read block 319 len 20480 18/30 Read block 277 len 20480 19/30 Write block 253 len 36864 data: 16 16 19 20 18 14 11 10 12 20/30 Read block 168 len 28672 21/30 Read block 469 len 32768 22/30 Write block 190 len 32768 data: 15 17 17 13 16 19 19 18 23/30 Flush 24/30 Write block 461 len 20480 data: 12 12 14 15 16 25/30 Read block 102 len 12288 26/30 Write block 581 len 20480 data: 13 12 13 16 15 27/30 Write block 41 len 16384 data: 21 19 18 17 28/30 Read block 542 len 24576 29/30 Write block 562 len 40960 data: 14 10 11 13 12 13 15 12 13 11 30/30 Read block 531 len 12288 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:10:57.430298703Z","hostname":"bmat-EVT22200007-0000589c","pid":1899} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 176 176 176 319 319 319 T T T 3 060-079 176 176 176 319 319 319 F F F 4 080-099 178 178 178 321 321 321 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 176 176 176 319 319 319 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 178 178 178 321 321 321 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 319 319 319 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 178 178 178 321 321 321 F F F 14 280-299 178 178 178 320 320 320 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 178 178 178 321 321 321 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 176 176 176 319 319 319 F F F 23 460-479 176 176 176 319 319 319 T T T 24 480-499 174 174 174 317 317 317 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 172 172 172 315 315 315 T T T 29 580-599 169 169 169 308 308 308 T T T Max gen: 178, Max flush: 321 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 179 179 179 322 322 322 F F F 3 060-079 176 176 176 319 319 319 F F F 4 080-099 178 178 178 321 321 321 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 176 176 176 319 319 319 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 178 178 178 321 321 321 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 176 176 176 319 319 319 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 178 178 178 321 321 321 F F F 14 280-299 178 178 178 320 320 320 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 172 172 172 315 315 315 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 178 178 178 321 321 321 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 176 176 176 319 319 319 F F F 23 460-479 179 179 179 322 322 322 F F F 24 480-499 174 174 174 317 317 317 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 179 179 179 322 322 322 F F F 29 580-599 179 179 179 322 322 322 F F F Max gen: 179, Max flush: 322 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.740649271Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.794047711Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.794780901Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.794835463Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.794854111Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.794871156Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"Crucible 231af610-8b7e-42ad-92c8-e924ff15db09 has session id: a73ac831-c6cf-45b6-acc3-4594651b5047","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.796132827Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Upstairs opts: Upstairs UUID: 231af610-8b7e-42ad-92c8-e924ff15db09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.796184683Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Crucible stats registered with UUID: 231af610-8b7e-42ad-92c8-e924ff15db09","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.796203612Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:01.796218612Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797200271Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797230503Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"231af610-8b7e-42ad-92c8-e924ff15db09 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797352272Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797374648Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797405651Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797430602Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797448689Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797464642Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797481867Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797520937Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797638447Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797661244Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"ds_connection connected from Ok(127.0.0.1:45460)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.797736728Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"ds_connection connected from Ok(127.0.0.1:51639)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.79800163Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"ds_connection connected from Ok(127.0.0.1:51683)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.798062745Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.799100078Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.799167977Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.799503233Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.799887409Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.800052727Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.800400468Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.800632523Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.800746186Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[0]R flush_numbers[0..12]: [321, 318, 322, 319, 321, 319, 319, 318, 321, 321, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.80117835Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[0]R generation[0..12]: [178, 175, 179, 176, 178, 176, 176, 175, 178, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.80120793Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801226518Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[1]R flush_numbers[0..12]: [321, 318, 322, 319, 321, 319, 319, 318, 321, 321, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801243453Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[1]R generation[0..12]: [178, 175, 179, 176, 178, 176, 176, 175, 178, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801259606Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.8012764Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[2]R flush_numbers[0..12]: [321, 318, 322, 319, 321, 319, 319, 318, 321, 321, 319, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801293616Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[2]R generation[0..12]: [178, 175, 179, 176, 178, 176, 176, 175, 178, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.80131048Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801327044Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801342345Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801365132Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801381044Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801396757Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"":"downstairs","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801412379Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-23T06:11:01.801426948Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"0","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801456288Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"1","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801472181Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"client":"2","":"downstairs","session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801487933Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801502363Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"231af610-8b7e-42ad-92c8-e924ff15db09 is now active with session: a73ac831-c6cf-45b6-acc3-4594651b5047","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801517764Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801531833Z","hostname":"bmat-EVT22200007-0000589c","pid":1907,"session_id":"a73ac831-c6cf-45b6-acc3-4594651b5047"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.801546242Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} {"msg":"Activated sub_volume 231af610-8b7e-42ad-92c8-e924ff15db09","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:01.80155977Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 594 len 8192 data: 12 10 03/30 Write block 361 len 12288 data: 15 14 15 04/30 Read block 247 len 28672 05/30 Read block 303 len 4096 06/30 Read block 231 len 8192 07/30 Read block 361 len 24576 08/30 Read block 159 len 16384 09/30 Write block 76 len 28672 data: 9 9 13 14 16 16 15 10/30 Write block 214 len 16384 data: 5 5 7 12 11/30 Read block 371 len 28672 12/30 Flush 13/30 Write block 327 len 8192 data: 10 12 14/30 Read block 182 len 4096 15/30 Read block 170 len 32768 16/30 Read block 169 len 40960 17/30 Read block 495 len 32768 18/30 Read block 179 len 20480 19/30 Write block 170 len 28672 data: 19 20 18 18 18 17 15 20/30 Read block 186 len 12288 21/30 Flush 22/30 Write block 49 len 36864 data: 14 14 13 18 20 19 17 14 18 23/30 Read block 300 len 12288 24/30 Read block 584 len 8192 25/30 Flush 26/30 Write block 293 len 20480 data: 14 18 18 19 21 27/30 Write block 123 len 16384 data: 8 13 13 14 28/30 Read block 311 len 28672 29/30 Write block 478 len 20480 data: 14 14 15 13 14 30/30 Read block 366 len 24576 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:01.829512009Z","hostname":"bmat-EVT22200007-0000589c","pid":1907} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 180 180 180 326 326 326 F F F 3 060-079 180 180 180 324 324 324 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 176 176 176 319 319 319 T T T 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 180 180 180 324 324 324 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 178 178 178 321 321 321 F F F 14 280-299 178 178 178 320 320 320 T T T 15 300-319 176 176 176 319 319 319 F F F 16 320-339 180 180 180 325 325 325 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 178 178 178 321 321 321 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 176 176 176 319 319 319 F F F 23 460-479 179 179 179 322 322 322 T T T 24 480-499 174 174 174 317 317 317 T T T 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 179 179 179 322 322 322 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 180, Max flush: 326 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 175 175 175 318 318 318 F F F 2 040-059 180 180 180 326 326 326 F F F 3 060-079 180 180 180 324 324 324 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 181 181 181 327 327 327 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 180 180 180 324 324 324 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 178 178 178 321 321 321 F F F 13 260-279 178 178 178 321 321 321 F F F 14 280-299 181 181 181 327 327 327 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 180 180 180 325 325 325 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 178 178 178 321 321 321 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 176 176 176 319 319 319 F F F 23 460-479 181 181 181 327 327 327 F F F 24 480-499 181 181 181 327 327 327 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 179 179 179 322 322 322 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 181, Max flush: 327 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.140646311Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.195225468Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.196147203Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.196200993Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.196219952Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.196237258Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"Crucible 3ecd93f2-6122-444a-acba-4218fa686c69 has session id: b6548f8e-a6d1-45c3-afb9-6ef9438a8e74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.197343481Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Upstairs opts: Upstairs UUID: 3ecd93f2-6122-444a-acba-4218fa686c69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.197392411Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Crucible stats registered with UUID: 3ecd93f2-6122-444a-acba-4218fa686c69","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.197413434Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:06.197492045Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.198402247Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.198434944Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"3ecd93f2-6122-444a-acba-4218fa686c69 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.198565571Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.198590753Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.19925395Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199311347Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199331589Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199348062Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199429228Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.19944958Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199466675Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199483018Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"ds_connection connected from Ok(127.0.0.1:48357)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199499662Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"ds_connection connected from Ok(127.0.0.1:59143)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.199516116Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"ds_connection connected from Ok(127.0.0.1:35678)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.19953282Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.200146436Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.200216739Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.200592829Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.200786796Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.200928907Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.20136087Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.201588486Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.201689753Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[0]R flush_numbers[0..12]: [321, 318, 326, 324, 324, 319, 327, 318, 325, 321, 324, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202184616Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[0]R generation[0..12]: [178, 175, 180, 180, 180, 176, 181, 175, 180, 178, 180, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202225048Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.20224531Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[1]R flush_numbers[0..12]: [321, 318, 326, 324, 324, 319, 327, 318, 325, 321, 324, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202263116Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[1]R generation[0..12]: [178, 175, 180, 180, 180, 176, 181, 175, 180, 178, 180, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202280261Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202297116Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[2]R flush_numbers[0..12]: [321, 318, 326, 324, 324, 319, 327, 318, 325, 321, 324, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202319872Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[2]R generation[0..12]: [178, 175, 180, 180, 180, 176, 181, 175, 180, 178, 180, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202337128Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202354102Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:11:06.202458025Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202486674Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202502747Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202518579Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"":"downstairs","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202534251Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202620748Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"0","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202734761Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"1","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-23T06:11:06.202813903Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"client":"2","":"downstairs","session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.2029198Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202936314Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"3ecd93f2-6122-444a-acba-4218fa686c69 is now active with session: b6548f8e-a6d1-45c3-afb9-6ef9438a8e74","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202951795Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202966074Z","hostname":"bmat-EVT22200007-0000589c","pid":1915,"session_id":"b6548f8e-a6d1-45c3-afb9-6ef9438a8e74"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202984783Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} {"msg":"Activated sub_volume 3ecd93f2-6122-444a-acba-4218fa686c69","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:06.202998501Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 328 len 16384 data: 13 14 17 17 02/30 Read block 145 len 24576 03/30 Read block 487 len 20480 04/30 Write block 457 len 40960 data: 12 12 11 9 13 13 15 16 17 15 05/30 Write block 503 len 4096 data: 17 06/30 Write block 579 len 28672 data: 11 12 14 13 14 17 16 07/30 Write block 210 len 32768 data: 13 8 9 7 6 6 8 13 08/30 Read block 98 len 28672 09/30 Write block 410 len 24576 data: 18 18 16 14 13 11 10/30 Read block 579 len 16384 11/30 Read block 99 len 12288 12/30 Flush 13/30 Write block 473 len 20480 data: 11 5 6 11 12 14/30 Write block 87 len 20480 data: 12 13 12 16 13 15/30 Write block 205 len 20480 data: 14 15 12 13 12 16/30 Read block 584 len 28672 17/30 Read block 25 len 16384 18/30 Read block 567 len 8192 19/30 Write block 56 len 40960 data: 15 19 17 16 19 17 15 14 12 13 20/30 Flush 21/30 Write block 459 len 8192 data: 12 10 22/30 Write block 241 len 8192 data: 13 14 23/30 Write block 39 len 32768 data: 20 20 22 20 19 18 15 15 24/30 Read block 430 len 40960 25/30 Read block 544 len 32768 26/30 Read block 393 len 24576 27/30 Read block 81 len 4096 28/30 Read block 339 len 16384 29/30 Read block 415 len 32768 30/30 Write block 27 len 36864 data: 12 12 11 12 12 13 13 19 19 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:06.228889783Z","hostname":"bmat-EVT22200007-0000589c","pid":1915} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 175 175 175 318 318 318 T T T 2 040-059 182 182 182 329 329 329 T T T 3 060-079 182 182 182 329 329 329 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 181 181 181 327 327 327 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 182 182 182 329 329 329 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 178 178 178 321 321 321 T T T 13 260-279 178 178 178 321 321 321 F F F 14 280-299 181 181 181 327 327 327 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 182 182 182 328 328 328 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 182 182 182 328 328 328 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 182 182 182 328 328 328 T T T 23 460-479 182 182 182 329 329 329 T T T 24 480-499 181 181 181 327 327 327 F F F 25 500-519 182 182 182 328 328 328 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 182 182 182 328 328 328 F F F 29 580-599 182 182 182 328 328 328 F F F Max gen: 182, Max flush: 329 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 182 182 182 329 329 329 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 176 176 176 319 319 319 F F F 6 120-139 181 181 181 327 327 327 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 178 178 178 321 321 321 F F F 10 200-219 182 182 182 329 329 329 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 183 183 183 330 330 330 F F F 13 260-279 178 178 178 321 321 321 F F F 14 280-299 181 181 181 327 327 327 F F F 15 300-319 176 176 176 319 319 319 F F F 16 320-339 182 182 182 328 328 328 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 168 168 168 306 306 306 F F F 20 400-419 182 182 182 328 328 328 F F F 21 420-439 175 175 175 318 318 318 F F F 22 440-459 183 183 183 330 330 330 F F F 23 460-479 183 183 183 330 330 330 F F F 24 480-499 181 181 181 327 327 327 F F F 25 500-519 182 182 182 328 328 328 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 182 182 182 328 328 328 F F F 29 580-599 182 182 182 328 328 328 F F F Max gen: 183, Max flush: 330 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.554431362Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.608432656Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.609215678Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.609309129Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.609337708Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.609356787Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"Crucible 67fa2629-3835-4cfa-8876-35a7391cc898 has session id: bb4811a4-9a97-4914-907a-c884465b9bce","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.610221025Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Upstairs opts: Upstairs UUID: 67fa2629-3835-4cfa-8876-35a7391cc898, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.610270286Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Crucible stats registered with UUID: 67fa2629-3835-4cfa-8876-35a7391cc898","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.610288523Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:10.610303854Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611043257Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611074281Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"67fa2629-3835-4cfa-8876-35a7391cc898 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611224939Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611246012Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611276544Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611293218Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611310474Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611326266Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611343561Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611365767Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611383193Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611399446Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"ds_connection connected from Ok(127.0.0.1:60859)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611686613Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"ds_connection connected from Ok(127.0.0.1:36746)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.611717296Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"ds_connection connected from Ok(127.0.0.1:57499)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.612232851Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.613068531Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.613145558Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.613450001Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.61373795Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.613969684Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.614271993Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615047459Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615264133Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[0]R flush_numbers[0..12]: [321, 330, 330, 329, 329, 319, 327, 318, 325, 321, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615405432Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[0]R generation[0..12]: [178, 183, 183, 182, 182, 176, 181, 175, 180, 178, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.61544875Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615481197Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[1]R flush_numbers[0..12]: [321, 330, 330, 329, 329, 319, 327, 318, 325, 321, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615511609Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[1]R generation[0..12]: [178, 183, 183, 182, 182, 176, 181, 175, 180, 178, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615540628Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615569858Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[2]R flush_numbers[0..12]: [321, 330, 330, 329, 329, 319, 327, 318, 325, 321, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615599689Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[2]R generation[0..12]: [178, 183, 183, 182, 182, 176, 181, 175, 180, 178, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615628348Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615657538Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615684182Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615711348Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615738062Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615762392Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"":"downstairs","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615778305Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615792343Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"0","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615812334Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"1","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615828237Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"client":"2","":"downstairs","session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615844099Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615858749Z","hostname":"bmat-EVT22200007-0000589c","pid":1923Wait for a query_work_queue command to finish before sending IO ,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"67fa2629-3835-4cfa-8876-35a7391cc898 is now active with session: bb4811a4-9a97-4914-907a-c884465b9bce","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615889121Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615903451Z","hostname":"bmat-EVT22200007-0000589c","pid":1923,"session_id":"bb4811a4-9a97-4914-907a-c884465b9bce"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615918101Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} {"msg":"Activated sub_volume 67fa2629-3835-4cfa-8876-35a7391cc898","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:10.615931478Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 28672 data: 15 19 19 20 22 17 14 02/30 Read block 452 len 24576 03/30 Write block 185 len 40960 data: 15 17 16 12 12 16 18 18 14 17 04/30 Write block 271 len 16384 data: 12 8 9 12 05/30 Read block 76 len 4096 06/30 Write block 499 len 40960 data: 12 14 16 15 18 17 16 18 18 18 07/30 Flush 08/30 Write block 407 len 36864 data: 16 20 19 19 19 17 15 14 12 09/30 Read block 562 len 36864 10/30 Write block 390 len 8192 data: 18 22 11/30 Read block 132 len 36864 12/30 Read block 55 len 4096 13/30 Read block 295 len 20480 14/30 Read block 167 len 32768 15/30 Flush 16/30 Flush 17/30 Flush 18/30 Write block 109 len 36864 data: 17 19 17 18 21 19 18 20 17 19/30 Flush 20/30 Write block 478 len 4096 data: 15 21/30 Read block 165 len 28672 22/30 Write block 134 len 12288 data: 13 12 12 23/30 Write block 464 len 12288 data: 17 18 16 24/30 Read block 454 len 40960 25/30 Read block 428 len 8192 26/30 Read block 419 len 40960 27/30 Write block 296 len 28672 data: 21 23 18 15 15 12 11 28/30 Write block 269 len 16384 data: 14 13 13 9 29/30 Write block 1 len 24576 data: 5 8 10 10 13 15 30/30 Write block 426 len 16384 data: 8 8 10 14 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:10.642697089Z","hostname":"bmat-EVT22200007-0000589c","pid":1923} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 321 321 321 T T T 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 182 182 182 329 329 329 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 181 181 181 327 327 327 T T T 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 184 184 184 331 331 331 F F F 10 200-219 182 182 182 329 329 329 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 183 183 183 330 330 330 F F F 13 260-279 184 184 184 331 331 331 T T T 14 280-299 184 184 184 331 331 331 T T T 15 300-319 176 176 176 319 319 319 T T T 16 320-339 182 182 182 328 328 328 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 184 184 184 332 332 332 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 175 175 175 318 318 318 T T T 22 440-459 183 183 183 330 330 330 F F F 23 460-479 183 183 183 330 330 330 T T T 24 480-499 184 184 184 331 331 331 F F F 25 500-519 184 184 184 331 331 331 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 182 182 182 328 328 328 F F F 29 580-599 182 182 182 328 328 328 F F F Max gen: 184, Max flush: 335 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 336 336 336 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 182 182 182 329 329 329 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 185 185 185 336 336 336 F F F 7 140-159 175 175 175 318 318 318 F F F 8 160-179 180 180 180 325 325 325 F F F 9 180-199 184 184 184 331 331 331 F F F 10 200-219 182 182 182 329 329 329 F F F 11 220-239 172 172 172 315 315 315 F F F 12 240-259 183 183 183 330 330 330 F F F 13 260-279 185 185 185 336 336 336 F F F 14 280-299 185 185 185 336 336 336 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 182 182 182 328 328 328 F F F 17 340-359 176 176 176 319 319 319 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 184 184 184 332 332 332 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 185 185 185 336 336 336 F F F 22 440-459 183 183 183 330 330 330 F F F 23 460-479 185 185 185 336 336 336 F F F 24 480-499 184 184 184 331 331 331 F F F 25 500-519 184 184 184 331 331 331 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 178 178 178 320 320 320 F F F 28 560-579 182 182 182 328 328 328 F F F 29 580-599 182 182 182 328 328 328 F F F Max gen: 185, Max flush: 336 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:14.97501423Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03018164Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.031170363Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.031224433Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.031245416Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.031263403Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"Crucible 699ba66f-ea24-4094-83fd-e43874b835b5 has session id: 14ab96ba-81d8-4556-8ad7-58e591426cf2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.032066426Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Upstairs opts: Upstairs UUID: 699ba66f-ea24-4094-83fd-e43874b835b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.032172323Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Crucible stats registered with UUID: 699ba66f-ea24-4094-83fd-e43874b835b5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.032197414Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:15.032286376Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033068296Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033097676Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"699ba66f-ea24-4094-83fd-e43874b835b5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033223012Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033251701Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033282113Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033298797Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033315972Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033400265Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033427581Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033443864Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033460658Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033477022Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62232)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033494377Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42796)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033811566Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54109)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.033915148Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.034511779Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.034545829Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.034711257Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.035321786Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.035589033Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.035896302Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036152185Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036173779Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[0]R flush_numbers[0..12]: [336, 330, 330, 329, 329, 335, 336, 318, 325, 331, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036793197Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[0]R generation[0..12]: [185, 183, 183, 182, 182, 184, 185, 175, 180, 184, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036825072Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036843169Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[1]R flush_numbers[0..12]: [336, 330, 330, 329, 329, 335, 336, 318, 325, 331, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036862398Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[1]R generation[0..12]: [185, 183, 183, 182, 182, 184, 185, 175, 180, 184, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036878962Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036895656Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[2]R flush_numbers[0..12]: [336, 330, 330, 329, 329, 335, 336, 318, 325, 331, 329, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03691231Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[2]R generation[0..12]: [185, 183, 183, 182, 182, 184, 185, 175, 180, 184, 182, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036928624Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036953074Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.036969417Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03698557Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037028779Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037044591Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"":"downstairs","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037060173Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037074392Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"0","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037090545Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"1","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037106397Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"client":"2","":"downstairs","session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037122049Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037136439Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"699ba66f-ea24-4094-83fd-e43874b835b5 is now active with session: 14ab96ba-81d8-4556-8ad7-58e591426cf2","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03715175Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03716622Z","hostname":"bmat-EVT22200007-0000589c","pid":1931,"session_id":"14ab96ba-81d8-4556-8ad7-58e591426cf2"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.03718082Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} {"msg":"Activated sub_volume 699ba66f-ea24-4094-83fd-e43874b835b5","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:15.037194488Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 16384 data: 13 14 14 15 02/30 Write block 220 len 32768 data: 14 13 17 18 17 15 11 9 03/30 Write block 576 len 32768 data: 13 13 12 12 13 15 14 15 04/30 Flush 05/30 Write block 137 len 32768 data: 12 14 17 16 16 14 19 20 06/30 Write block 215 len 8192 data: 7 9 07/30 Write block 163 len 12288 data: 15 15 12 08/30 Write block 150 len 8192 data: 15 15 09/30 Read block 357 len 28672 10/30 Write block 379 len 28672 data: 17 15 15 13 10 10 12 11/30 Read block 324 len 40960 12/30 Read block 233 len 32768 13/30 Flush 14/30 Write block 1 len 16384 data: 6 9 11 11 15/30 Write block 513 len 20480 data: 16 16 16 15 16 16/30 Write block 83 len 28672 data: 15 16 16 13 13 14 13 17/30 Read block 130 len 8192 18/30 Write block 172 len 32768 data: 19 19 19 18 16 12 14 15 19/30 Read block 105 len 12288 20/30 Read block 358 len 36864 21/30 Write block 526 len 4096 data: 12 22/30 Write block 279 len 32768 data: 16 16 14 13 13 13 12 11 23/30 Write block 233 len 24576 data: 15 11 12 13 15 15 24/30 Read block 122 len 4096 25/30 Write block 78 len 24576 data: 14 15 17 17 16 16 26/30 Write block 542 len 8192 data: 12 11 27/30 Write block 331 len 40960 data: 18 17 18 17 14 14 15 14 15 16 28/30 Write block 329 len 32768 data: 15 18 19 18 19 18 15 15 29/30 Read block 529 len 32768 30/30 Read block 256 len 24576 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 17 16 17 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:15.064347732Z","hostname":"bmat-EVT22200007-0000589c","pid":1931} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 336 336 336 T T T 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 182 182 182 329 329 329 T T T 4 080-099 182 182 182 329 329 329 T T T 5 100-119 184 184 184 335 335 335 F F F 6 120-139 186 186 186 338 338 338 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 186 186 186 338 338 338 T T T 9 180-199 184 184 184 331 331 331 F F F 10 200-219 186 186 186 338 338 338 F F F 11 220-239 186 186 186 337 337 337 T T T 12 240-259 183 183 183 330 330 330 F F F 13 260-279 186 186 186 337 337 337 T T T 14 280-299 185 185 185 336 336 336 T T T 15 300-319 185 185 185 336 336 336 F F F 16 320-339 182 182 182 328 328 328 T T T 17 340-359 176 176 176 319 319 319 T T T 18 360-379 186 186 186 338 338 338 F F F 19 380-399 186 186 186 338 338 338 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 185 185 185 336 336 336 F F F 22 440-459 183 183 183 330 330 330 F F F 23 460-479 185 185 185 336 336 336 F F F 24 480-499 184 184 184 331 331 331 F F F 25 500-519 184 184 184 331 331 331 T T T 26 520-539 178 178 178 320 320 320 T T T 27 540-559 178 178 178 320 320 320 T T T 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 186, Max flush: 338 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 186 186 186 338 338 338 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 187 187 187 339 339 339 F F F 9 180-199 184 184 184 331 331 331 F F F 10 200-219 186 186 186 338 338 338 F F F 11 220-239 187 187 187 339 339 339 F F F 12 240-259 183 183 183 330 330 330 F F F 13 260-279 187 187 187 339 339 339 F F F 14 280-299 187 187 187 339 339 339 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 187 187 187 339 339 339 F F F 18 360-379 186 186 186 338 338 338 F F F 19 380-399 186 186 186 338 338 338 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 185 185 185 336 336 336 F F F 22 440-459 183 183 183 330 330 330 F F F 23 460-479 185 185 185 336 336 336 F F F 24 480-499 184 184 184 331 331 331 F F F 25 500-519 187 187 187 339 339 339 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 187 187 187 339 339 339 F F F 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 187, Max flush: 339 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.478029813Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.530586682Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.531331235Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.531410357Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.53143645Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.531454798Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"Crucible abc6d665-85a0-404f-ad33-72f14e62b528 has session id: 546d1e52-d024-4974-927c-3f5e3c44d4d3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.532368517Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Upstairs opts: Upstairs UUID: abc6d665-85a0-404f-ad33-72f14e62b528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.532479635Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Crucible stats registered with UUID: abc6d665-85a0-404f-ad33-72f14e62b528","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.532501279Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:19.532516901Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533336939Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.53336701Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"abc6d665-85a0-404f-ad33-72f14e62b528 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533499561Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533519713Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533551738Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533568533Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533761167Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533823815Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533843675Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533866171Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.533884188Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.534095721Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41292)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.534165384Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56593)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.534185324Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65275)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.53420266Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.535137412Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.535183246Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.535255223Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536065621Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536100232Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.5361187Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.5368762Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536933948Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[0]R flush_numbers[0..12]: [339, 330, 330, 339, 339, 335, 338, 338, 339, 331, 338, 339]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536953187Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[0]R generation[0..12]: [187, 183, 183, 187, 187, 184, 186, 186, 187, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536978669Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.536997107Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[1]R flush_numbers[0..12]: [339, 330, 330, 339, 339, 335, 338, 338, 339, 331, 338, 339]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537014242Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[1]R generation[0..12]: [187, 183, 183, 187, 187, 184, 186, 186, 187, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.53703264Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537050526Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[2]R flush_numbers[0..12]: [339, 330, 330, 339, 339, 335, 338, 338, 339, 331, 338, 339]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.53706708Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[2]R generation[0..12]: [187, 183, 183, 187, 187, 184, 186, 186, 187, 184, 186, 187]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:11:19.537159519Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537390592Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537413008Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537430614Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537446416Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537461567Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"":"downstairs","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537477029Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537493042Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"0","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537517061Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"1","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537533314Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"client":"2","":"downstairs","session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537548786Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537563406Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"abc6d665-85a0-404f-ad33-72f14e62b528 is now active with session: 546d1e52-d024-4974-927c-3f5e3c44d4d3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537578447Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537592836Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537605342Z","hostname":"bmat-EVT22200007-0000589c","pid":1939,"session_id":"546d1e52-d024-4974-927c-3f5e3c44d4d3"} {"msg":"Activated sub_volume abc6d665-85a0-404f-ad33-72f14e62b528","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:19.537619841Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 12288 data: 15 12 11 02/30 Write block 500 len 12288 data: 15 17 16 03/30 Flush 04/30 Write block 243 len 20480 data: 12 13 15 15 17 05/30 Read block 480 len 36864 06/30 Read block 224 len 4096 07/30 Read block 67 len 40960 08/30 Read block 529 len 24576 09/30 Write block 436 len 8192 data: 14 11 10/30 Read block 433 len 24576 11/30 Flush 12/30 Write block 236 len 36864 data: 14 16 16 12 12 14 15 13 14 13/30 Flush 14/30 Write block 10 len 16384 data: 13 12 16 16 15/30 Read block 311 len 12288 16/30 Read block 80 len 36864 17/30 Read block 2 len 12288 18/30 Read block 202 len 4096 19/30 Write block 134 len 20480 data: 14 13 13 13 15 20/30 Write block 356 len 12288 data: 16 18 17 21/30 Read block 396 len 20480 22/30 Read block 432 len 16384 23/30 Write block 257 len 20480 data: 19 16 13 12 13 24/30 Write block 449 len 12288 data: 15 16 13 25/30 Read block 178 len 16384 26/30 Write block 128 len 32768 data: 12 12 12 11 11 12 15 14 27/30 Read block 560 len 8192 28/30 Read block 258 len 40960 29/30 Read block 112 len 28672 30/30 Read block 154 len 8192 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 17 17 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:19.563241091Z","hostname":"bmat-EVT22200007-0000589c","pid":1939} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 T T T 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 186 186 186 338 338 338 T T T 7 140-159 186 186 186 338 338 338 F F F 8 160-179 187 187 187 339 339 339 F F F 9 180-199 184 184 184 331 331 331 F F F 10 200-219 186 186 186 338 338 338 F F F 11 220-239 188 188 188 342 342 342 F F F 12 240-259 188 188 188 342 342 342 T T T 13 260-279 188 188 188 340 340 340 T T T 14 280-299 187 187 187 339 339 339 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 187 187 187 339 339 339 T T T 18 360-379 186 186 186 338 338 338 F F F 19 380-399 186 186 186 338 338 338 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 188 188 188 341 341 341 F F F 22 440-459 183 183 183 330 330 330 T T T 23 460-479 185 185 185 336 336 336 F F F 24 480-499 184 184 184 331 331 331 F F F 25 500-519 188 188 188 340 340 340 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 187 187 187 339 339 339 F F F 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 188, Max flush: 342 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 343 343 343 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 189 189 189 343 343 343 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 187 187 187 339 339 339 F F F 9 180-199 184 184 184 331 331 331 F F F 10 200-219 186 186 186 338 338 338 F F F 11 220-239 188 188 188 342 342 342 F F F 12 240-259 189 189 189 343 343 343 F F F 13 260-279 189 189 189 343 343 343 F F F 14 280-299 187 187 187 339 339 339 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 189 189 189 343 343 343 F F F 18 360-379 186 186 186 338 338 338 F F F 19 380-399 186 186 186 338 338 338 F F F 20 400-419 184 184 184 332 332 332 F F F 21 420-439 188 188 188 341 341 341 F F F 22 440-459 189 189 189 343 343 343 F F F 23 460-479 185 185 185 336 336 336 F F F 24 480-499 184 184 184 331 331 331 F F F 25 500-519 188 188 188 340 340 340 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 187 187 187 339 339 339 F F F 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 189, Max flush: 343 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.885098732Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:23.936244433Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.936681778Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.973041268Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.973693833Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97374122Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.973758786Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.973776923Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"Crucible ce01f387-0700-4c02-b006-8e90ff168d81 has session id: 41fad5c9-2e66-4064-8476-7bb2100bece4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97482614Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Upstairs opts: Upstairs UUID: ce01f387-0700-4c02-b006-8e90ff168d81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.974882505Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Crucible stats registered with UUID: ce01f387-0700-4c02-b006-8e90ff168d81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.974903378Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:23.974919481Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975603721Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975671549Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"ce01f387-0700-4c02-b006-8e90ff168d81 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975689887Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975731211Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975765983Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.9757842Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975802026Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97581836Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975836347Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97585264Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975869935Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.975996504Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65029)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.976143936Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56338)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.976221024Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61570)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.976402194Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.977225879Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.977523518Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.977671702Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.978093014Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"2","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.978455365Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.978683562Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.978929285Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.978952362Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[0]R flush_numbers[0..12]: [343, 330, 330, 339, 339, 335, 343, 338, 339, 331, 338, 342]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979765525Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[0]R generation[0..12]: [189, 183, 183, 187, 187, 184, 189, 186, 187, 184, 186, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979789124Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979806349Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[1]R flush_numbers[0..12]: [343, 330, 330, 339, 339, 335, 343, 338, 339, 331, 338, 342]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979822893Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[1]R generation[0..12]: [189, 183, 183, 187, 187, 184, 189, 186, 187, 184, 186, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979839627Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979856041Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[2]R flush_numbers[0..12]: [343, 330, 330, 339, 339, 335, 343, 338, 339, 331, 338, 342]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979872214Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[2]R generation[0..12]: [189, 183, 183, 187, 187, 184, 189, 186, 187, 184, 186, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97989495Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979911855Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979927186Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979942638Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979957628Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.97997293Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"":"downstairs","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.979988442Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980003102Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"0","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980019114Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"1","":"downstairs","session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980034897Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"client":"2","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980064407Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980078806Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"ce01f387-0700-4c02-b006-8e90ff168d81 is now active with session: 41fad5c9-2e66-4064-8476-7bb2100bece4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980094058Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980108166Z","hostname":"bmat-EVT22200007-0000589c","pid":1947,"session_id":"41fad5c9-2e66-4064-8476-7bb2100bece4"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980122626Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} {"msg":"Activated sub_volume ce01f387-0700-4c02-b006-8e90ff168d81","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:23.980138669Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 8192 data: 15 14 02/30 Write block 410 len 24576 data: 20 20 18 16 15 13 03/30 Read block 443 len 12288 04/30 Write block 179 len 12288 data: 16 13 14 05/30 Read block 258 len 4096 06/30 Read block 346 len 32768 07/30 Write block 166 len 12288 data: 13 20 23 08/30 Read block 544 len 32768 09/30 Read block 117 len 4096 10/30 Write block 400 len 20480 data: 14 18 14 16 19 11/30 Write block 487 len 28672 data: 11 11 10 8 9 11 15 12/30 Write block 542 len 16384 data: 13 12 12 8 13/30 Read block 171 len 8192 14/30 Write block 479 len 12288 data: 15 16 14 15/30 Read block 261 len 8192 16/30 Flush 17/30 Write block 270 len 4096 data: 14 18/30 Write block 121 len 28672 data: 8 9 9 14 14 15 13 19/30 Read block 345 len 20480 20/30 Read block 8 len 28672 21/30 Write block 182 len 8192 data: 13 11 22/30 Read block 261 len 36864 23/30 Read block 535 len 32768 24/30 Write block 472 len 20480 data: 12 12 6 7 12 25/30 Read block 13 len 4096 26/30 Write block 206 len 24576 data: 16 13 14 13 14 9 27/30 Write block 378 len 12288 data: 16 18 16 28/30 Read block 558 len 12288 29/30 Write block 43 len 40960 data: 20 19 16 16 17 12 15 15 14 19 30/30 Write block 224 len 8192 data: 18 16 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:24.004940272Z","hostname":"bmat-EVT22200007-0000589c","pid":1947} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 343 343 343 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 183 183 183 330 330 330 T T T 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 189 189 189 343 343 343 T T T 7 140-159 186 186 186 338 338 338 F F F 8 160-179 190 190 190 344 344 344 F F F 9 180-199 190 190 190 344 344 344 T T T 10 200-219 186 186 186 338 338 338 T T T 11 220-239 190 190 190 344 344 344 T T T 12 240-259 189 189 189 343 343 343 F F F 13 260-279 189 189 189 343 343 343 T T T 14 280-299 187 187 187 339 339 339 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 189 189 189 343 343 343 F F F 18 360-379 186 186 186 338 338 338 T T T 19 380-399 186 186 186 338 338 338 T T T 20 400-419 190 190 190 344 344 344 F F F 21 420-439 188 188 188 341 341 341 F F F 22 440-459 189 189 189 343 343 343 F F F 23 460-479 190 190 190 344 344 344 T T T 24 480-499 190 190 190 344 344 344 F F F 25 500-519 188 188 188 340 340 340 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 190, Max flush: 344 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 343 343 343 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 191 191 191 345 345 345 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 190 190 190 344 344 344 F F F 9 180-199 191 191 191 345 345 345 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 191 191 191 345 345 345 F F F 12 240-259 189 189 189 343 343 343 F F F 13 260-279 191 191 191 345 345 345 F F F 14 280-299 187 187 187 339 339 339 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 189 189 189 343 343 343 F F F 18 360-379 191 191 191 345 345 345 F F F 19 380-399 191 191 191 345 345 345 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 188 188 188 341 341 341 F F F 22 440-459 189 189 189 343 343 343 F F F 23 460-479 191 191 191 345 345 345 F F F 24 480-499 190 190 190 344 344 344 F F F 25 500-519 188 188 188 340 340 340 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 186 186 186 337 337 337 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 191, Max flush: 345 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.391685329Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.445308419Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.446264285Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.446319999Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.446344669Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.446363849Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"Crucible 3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6 has session id: ac6be717-4ffd-4a40-943f-18b1ad8bb5d9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.447281065Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Upstairs opts: Upstairs UUID: 3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.447337711Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Crucible stats registered with UUID: 3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.44735649Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:28.447372142Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448251912Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448281632Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448412039Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448435056Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448467032Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448484057Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448804182Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448829453Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448849003Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448885829Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448904697Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.448921421Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56282)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.449345699Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52626)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.449421585Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:28.449445534Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:28.449465384Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:28.449483912Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.449498582Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.449696397Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.449722821Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.450730162Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.450822301Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.451592176Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:28.451661127Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.450529065Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33216)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.450615522Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.452116183Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.452967565Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[0]R flush_numbers[0..12]: [343, 330, 345, 339, 339, 335, 345, 338, 344, 345, 345, 345]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453785278Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[0]R generation[0..12]: [189, 183, 191, 187, 187, 184, 191, 186, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453812112Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.45383045Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[1]R flush_numbers[0..12]: [343, 330, 345, 339, 339, 335, 345, 338, 344, 345, 345, 345]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453848066Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[1]R generation[0..12]: [189, 183, 191, 187, 187, 184, 191, 186, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453865191Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453882266Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[2]R flush_numbers[0..12]: [343, 330, 345, 339, 339, 335, 345, 338, 344, 345, 345, 345]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453898569Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[2]R generation[0..12]: [189, 183, 191, 187, 187, 184, 191, 186, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453915113Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453931918Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453959514Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453975857Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.453991389Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454006901Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"":"downstairs","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454023164Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454037133Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"0","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454052655Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"1","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454068467Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"client":"2","":"downstairs","session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454083949Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454098318Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6 is now active with session: ac6be717-4ffd-4a40-943f-18b1ad8bb5d9","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.45411378Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454127758Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454142458Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"Activated sub_volume 3ab798ef-bb25-4902-ae4e-9d8d7aaad9b6","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:38.454229536Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 238 len 28672 data: 17 13 13 15 16 14 15 03/30 Read block 347 len 12288 04/30 Write block 6 len 36864 data: 16 17 16 16 14 13 17 17 15 05/30 Read block 545 len 24576 06/30 Write block 425 len 32768 data: 10 9 9 11 15 14 15 17 07/30 Write block 29 len 36864 data: 12 13 13 14 14 20 20 17 18 08/30 Read block 518 len 4096 09/30 Write block 364 len 20480 data: 17 21 21 15 16 10/30 Read block 99 len 24576 11/30 Read block 313 len 40960 12/30 Read block 232 len 4096 13/30 Write block 84 len 32768 data: 17 17 14 14 15 14 17 14 14/30 Read block 95 len 36864 15/30 Read block 509 len 32768 16/30 Read block 184 len 12288 17/30 Read block 52 len 40960 18/30 Read block 424 len 24576 19/30 Write block 10 len 12288 data: 15 14 18 20/30 Write block 439 len 32768 data: 13 12 11 13 12 15 14 17 21/30 Read block 29 len 28672 22/30 Write block 501 len 32768 data: 18 17 19 18 17 19 19 19 23/30 Read block 284 len 20480 24/30 Write block 13 len 12288 data: 18 16 13 25/30 Read block 510 len 36864 26/30 Write block 573 len 24576 data: 10 13 12 14 14 13 27/30 Read block 485 len 28672 28/30 Write block 278 len 40960 data: 13 17 17 15 14 14 14 13 12 11 29/30 Write block 233 len 36864 data: 16 12 13 15 17 18 14 14 16 30/30 Write block 244 len 20480 data: 16 16 16 18 16 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Done Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 27 26 26 79 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 Wrote out file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486173152Z","hostname":"bmat-EVT22200007-0000589c","pid":1955} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486493838Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486823111Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486854095Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486870258Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486885238Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486899778Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486914248Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486929018Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486943728Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486958248Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486972707Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.486987297Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487001907Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487016748Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487032359Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48704741Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487062712Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487077863Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487092933Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487107854Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487125971Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487141112Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487156233Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487171294Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487186405Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487201245Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487216246Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487231256Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487246257Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487261148Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487276208Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487291119Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.4873061Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487321431Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487336432Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487351282Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487366243Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487383969Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48739933Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487414461Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487429592Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487444553Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487459714Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487474725Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487489775Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487504906Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487519837Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487534848Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487549868Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487565079Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48758012Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487595211Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487610322Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487625383Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487643249Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48765846Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487673581Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487688682Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487703643Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487718603Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487733614Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487748535Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487763515Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487778556Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487793517Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487808678Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487823709Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487838489Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487853469Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48786838Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487883421Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487898401Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487916007Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487931249Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487946369Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48796149Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487976531Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.487991372Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488006573Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488021844Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488036734Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488051795Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488066726Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488081716Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488097469Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488112369Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48812737Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48814212Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488156209Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488173424Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488187623Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488201742Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488215871Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488229809Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488243828Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488257706Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488271705Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488285664Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488299722Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488313751Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.4883279Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488342069Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488356018Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488369796Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488383814Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488397863Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488414708Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488429107Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488443336Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488457195Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488471173Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488485242Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.48849911Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488513129Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:38.488527027Z","hostname":"bmat-EVT22200007-0000589c","pid":1955,"session_id":"ac6be717-4ffd-4a40-943f-18b1ad8bb5d9"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 343 343 343 T T T 1 020-039 183 183 183 330 330 330 T T T 2 040-059 191 191 191 345 345 345 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 187 187 187 339 339 339 T T T 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 190 190 190 344 344 344 F F F 9 180-199 191 191 191 345 345 345 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 191 191 191 345 345 345 T T T 12 240-259 189 189 189 343 343 343 T T T 13 260-279 191 191 191 345 345 345 T T T 14 280-299 187 187 187 339 339 339 T T T 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 189 189 189 343 343 343 F F F 18 360-379 191 191 191 345 345 345 T T T 19 380-399 191 191 191 345 345 345 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 188 188 188 341 341 341 T T T 22 440-459 189 189 189 343 343 343 T T T 23 460-479 191 191 191 345 345 345 F F F 24 480-499 190 190 190 344 344 344 F F F 25 500-519 188 188 188 340 340 340 T T T 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 186 186 186 337 337 337 T T T 29 580-599 186 186 186 337 337 337 F F F Max gen: 191, Max flush: 345 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 192 192 192 346 346 346 F F F 2 040-059 191 191 191 345 345 345 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 190 190 190 344 344 344 F F F 9 180-199 191 191 191 345 345 345 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 192 192 192 346 346 346 F F F 12 240-259 192 192 192 346 346 346 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 F F F 17 340-359 189 189 189 343 343 343 F F F 18 360-379 192 192 192 346 346 346 F F F 19 380-399 191 191 191 345 345 345 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 192 192 192 346 346 346 F F F 22 440-459 192 192 192 346 346 346 F F F 23 460-479 191 191 191 345 345 345 F F F 24 480-499 190 190 190 344 344 344 F F F 25 500-519 192 192 192 346 346 346 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 192 192 192 346 346 346 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 192, Max flush: 346 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.857598886Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.912418284Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.913455115Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.913509457Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.913530209Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.913550471Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"Crucible efbb1cd2-891b-4221-8a60-163ddaf1ea91 has session id: 225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.91440057Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Upstairs opts: Upstairs UUID: efbb1cd2-891b-4221-8a60-163ddaf1ea91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.914451374Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Crucible stats registered with UUID: efbb1cd2-891b-4221-8a60-163ddaf1ea91","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.914477167Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:42.914496767Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.91525598Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915290671Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"efbb1cd2-891b-4221-8a60-163ddaf1ea91 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915421819Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915444676Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915478625Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915497684Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915516914Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.91553478Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915553839Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915641058Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915658904Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.915675057Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38137)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.916007317Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40344)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.916339657Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36637)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.916391493Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.916725116Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.916899743Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.917399836Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.917554322Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.918150492Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.918252481Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.918359359Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919023157Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[0]R flush_numbers[0..12]: [346, 346, 345, 339, 346, 335, 345, 338, 344, 345, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919780938Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[0]R generation[0..12]: [192, 192, 191, 187, 192, 184, 191, 186, 190, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919808995Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919827483Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[1]R flush_numbers[0..12]: [346, 346, 345, 339, 346, 335, 345, 338, 344, 345, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919844929Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[1]R generation[0..12]: [192, 192, 191, 187, 192, 184, 191, 186, 190, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919870752Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919889119Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[2]R flush_numbers[0..12]: [346, 346, 345, 339, 346, 335, 345, 338, 344, 345, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919906425Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[2]R generation[0..12]: [192, 192, 191, 187, 192, 184, 191, 186, 190, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919923279Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919940604Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Max found gen is 193","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919956687Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Generation requested: 194 >= found:193","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.91997275Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Next flush: 347","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.919988512Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920031621Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"":"downstairs","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920048044Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920062604Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"0","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920078667Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"1","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920094539Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"client":"2","":"downstairs","session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920110572Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920125874Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"efbb1cd2-891b-4221-8a60-163ddaf1ea91 is now active with session: 225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920145424Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920160144Z","hostname":"bmat-EVT22200007-0000589c","pid":1963,"session_id":"225cdb4c-8ad8-44d3-a739-2fdfbbaa8b9d"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920175174Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} {"msg":"Activated sub_volume efbb1cd2-891b-4221-8a60-163ddaf1ea91","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:42.920189123Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 457 len 12288 data: 13 13 13 03/30 Read block 30 len 8192 04/30 Read block 333 len 24576 05/30 Write block 510 len 24576 data: 20 19 18 17 17 17 06/30 Read block 111 len 36864 07/30 Write block 38 len 32768 data: 21 21 21 23 21 21 20 17 08/30 Write block 485 len 28672 data: 11 11 12 12 11 9 10 09/30 Write block 373 len 12288 data: 17 20 19 10/30 Read block 270 len 8192 11/30 Read block 9 len 40960 12/30 Write block 34 len 28672 data: 21 21 18 19 22 22 22 13/30 Write block 378 len 24576 data: 17 19 17 16 14 11 14/30 Write block 179 len 12288 data: 17 14 15 15/30 Read block 200 len 8192 16/30 Write block 251 len 20480 data: 18 15 17 17 20 17/30 Read block 540 len 8192 18/30 Write block 352 len 28672 data: 14 12 13 15 17 19 18 19/30 Write block 320 len 36864 data: 18 16 14 10 11 14 10 11 14 20/30 Read block 561 len 8192 21/30 Read block 353 len 12288 22/30 Read block 314 len 32768 23/30 Write block 485 len 8192 data: 12 12 24/30 Read block 64 len 40960 25/30 Read block 358 len 36864 26/30 Read block 298 len 32768 27/30 Read block 298 len 4096 28/30 Write block 438 len 8192 data: 13 14 29/30 Read block 275 len 12288 30/30 Read block 277 len 40960 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:42.94798429Z","hostname":"bmat-EVT22200007-0000589c","pid":1963} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 192 192 192 346 346 346 T T T 2 040-059 191 191 191 345 345 345 T T T 3 060-079 187 187 187 339 339 339 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 190 190 190 344 344 344 T T T 9 180-199 191 191 191 345 345 345 T T T 10 200-219 191 191 191 345 345 345 F F F 11 220-239 192 192 192 346 346 346 F F F 12 240-259 192 192 192 346 346 346 T T T 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 187 187 187 339 339 339 T T T 17 340-359 189 189 189 343 343 343 T T T 18 360-379 192 192 192 346 346 346 T T T 19 380-399 191 191 191 345 345 345 T T T 20 400-419 190 190 190 344 344 344 F F F 21 420-439 192 192 192 346 346 346 T T T 22 440-459 192 192 192 346 346 346 T T T 23 460-479 191 191 191 345 345 345 F F F 24 480-499 190 190 190 344 344 344 T T T 25 500-519 192 192 192 346 346 346 T T T 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 192 192 192 346 346 346 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 192, Max flush: 346 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 193 193 193 347 347 347 F F F 2 040-059 193 193 193 347 347 347 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 193 193 193 347 347 347 F F F 9 180-199 193 193 193 347 347 347 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 192 192 192 346 346 346 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 193 193 193 347 347 347 F F F 17 340-359 193 193 193 347 347 347 F F F 18 360-379 193 193 193 347 347 347 F F F 19 380-399 193 193 193 347 347 347 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 193 193 193 347 347 347 F F F 22 440-459 193 193 193 347 347 347 F F F 23 460-479 191 191 191 345 345 345 F F F 24 480-499 193 193 193 347 347 347 F F F 25 500-519 193 193 193 347 347 347 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 F F F 28 560-579 192 192 192 346 346 346 F F F 29 580-599 186 186 186 337 337 337 F F F Max gen: 193, Max flush: 347 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.329406738Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.382463048Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.383239958Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.383296323Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.383316194Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.383334511Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"Crucible 063edea7-7eb9-421f-b9cf-9fbcee55e154 has session id: 836e9c5a-ce88-4dda-b26d-e3b6e89428a4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.384270586Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Upstairs opts: Upstairs UUID: 063edea7-7eb9-421f-b9cf-9fbcee55e154, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.384318184Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Crucible stats registered with UUID: 063edea7-7eb9-421f-b9cf-9fbcee55e154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.384337113Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:47.384353185Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385178534Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.38522557Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"063edea7-7eb9-421f-b9cf-9fbcee55e154 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385275032Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385312679Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385358091Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385389395Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385420319Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385446342Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385900641Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385940393Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385968661Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.385995375Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62138)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.386022581Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52946)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.386050107Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64063)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.386077313Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.386790322Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.386888964Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.387480965Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.387533322Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.388195907Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.388288527Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.388310051Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389092151Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[0]R flush_numbers[0..12]: [346, 347, 347, 339, 346, 335, 345, 338, 347, 347, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389876085Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[0]R generation[0..12]: [192, 193, 193, 187, 192, 184, 191, 186, 193, 193, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389898691Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389915686Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[1]R flush_numbers[0..12]: [346, 347, 347, 339, 346, 335, 345, 338, 347, 347, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.38993217Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[1]R generation[0..12]: [192, 193, 193, 187, 192, 184, 191, 186, 193, 193, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389949265Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389965919Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[2]R flush_numbers[0..12]: [346, 347, 347, 339, 346, 335, 345, 338, 347, 347, 345, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389982843Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[2]R generation[0..12]: [192, 193, 193, 187, 192, 184, 191, 186, 193, 193, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.389998926Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390016172Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390037054Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Generation requested: 196 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390053278Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390068439Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.39008359Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"":"downstairs","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390099382Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390113631Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"0","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390129504Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"1","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390145095Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"client":"2","":"downstairs","session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390160437Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390175037Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"063edea7-7eb9-421f-b9cf-9fbcee55e154 is now active with session: 836e9c5a-ce88-4dda-b26d-e3b6e89428a4","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390189777Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390204056Z","hostname":"bmat-EVT22200007-0000589c","pid":1971,"session_id":"836e9c5a-ce88-4dda-b26d-e3b6e89428a4"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390218706Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} {"msg":"Activated sub_volume 063edea7-7eb9-421f-b9cf-9fbcee55e154","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:47.390231923Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 24576 data: 14 15 15 22 22 19 02/30 Read block 435 len 32768 03/30 Read block 300 len 28672 04/30 Write block 575 len 32768 data: 13 15 15 14 13 14 16 15 05/30 Write block 430 len 32768 data: 15 16 18 18 15 16 15 12 06/30 Write block 547 len 8192 data: 14 15 07/30 Read block 531 len 40960 08/30 Read block 403 len 32768 09/30 Read block 281 len 24576 10/30 Read block 427 len 8192 11/30 Write block 47 len 8192 data: 18 13 12/30 Write block 228 len 20480 data: 11 10 11 15 16 13/30 Read block 276 len 12288 14/30 Read block 402 len 16384 15/30 Write block 362 len 12288 data: 15 16 18 16/30 Write block 37 len 20480 data: 20 23 23 23 24 17/30 Write block 316 len 24576 data: 11 11 14 15 19 17 18/30 Write block 348 len 8192 data: 17 14 19/30 Write block 340 len 12288 data: 17 17 21 20/30 Read block 62 len 20480 21/30 Read block 217 len 4096 22/30 Read block 437 len 8192 23/30 Write block 428 len 32768 data: 12 16 16 17 19 19 16 17 24/30 Read block 112 len 40960 25/30 Read block 216 len 12288 26/30 Read block 371 len 40960 27/30 Read block 531 len 12288 28/30 Read block 448 len 28672 29/30 Read block 505 len 8192 30/30 Read block 297 len 8192 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:47.416906809Z","hostname":"bmat-EVT22200007-0000589c","pid":1971} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 193 193 193 347 347 347 T T T 2 040-059 193 193 193 347 347 347 T T T 3 060-079 187 187 187 339 339 339 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 193 193 193 347 347 347 F F F 9 180-199 193 193 193 347 347 347 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 192 192 192 346 346 346 T T T 12 240-259 193 193 193 347 347 347 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 185 185 185 336 336 336 T T T 16 320-339 193 193 193 347 347 347 T T T 17 340-359 193 193 193 347 347 347 T T T 18 360-379 193 193 193 347 347 347 T T T 19 380-399 193 193 193 347 347 347 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 193 193 193 347 347 347 T T T 22 440-459 193 193 193 347 347 347 F F F 23 460-479 191 191 191 345 345 345 F F F 24 480-499 193 193 193 347 347 347 F F F 25 500-519 193 193 193 347 347 347 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 190 190 190 344 344 344 T T T 28 560-579 192 192 192 346 346 346 T T T 29 580-599 186 186 186 337 337 337 T T T Max gen: 193, Max flush: 347 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 194 194 194 348 348 348 F F F 2 040-059 194 194 194 348 348 348 F F F 3 060-079 187 187 187 339 339 339 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 193 193 193 347 347 347 F F F 9 180-199 193 193 193 347 347 347 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 194 194 194 348 348 348 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 194 194 194 348 348 348 F F F 16 320-339 194 194 194 348 348 348 F F F 17 340-359 194 194 194 348 348 348 F F F 18 360-379 194 194 194 348 348 348 F F F 19 380-399 193 193 193 347 347 347 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 194 194 194 348 348 348 F F F 22 440-459 193 193 193 347 347 347 F F F 23 460-479 191 191 191 345 345 345 F F F 24 480-499 193 193 193 347 347 347 F F F 25 500-519 193 193 193 347 347 347 F F F 26 520-539 187 187 187 339 339 339 F F F 27 540-559 194 194 194 348 348 348 F F F 28 560-579 194 194 194 348 348 348 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 194, Max flush: 348 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.779593915Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.833056516Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.833734393Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a0baefa8a6d4c0e5f0d4f746f2af5de9c08b52b6\",\n git_commit_timestamp: \"2025-03-23T05:46:50.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.85.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"4eb161250e340c8f48f66e2b929ef4a5bed7c181\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.83378646Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.833811672Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.833829518Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"Crucible 32c88801-2598-479f-8a94-2e3c3cdca30f has session id: bf13df08-dccc-4ada-b01f-a4f984713413","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.834791166Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Upstairs opts: Upstairs UUID: 32c88801-2598-479f-8a94-2e3c3cdca30f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.834901391Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Crucible stats registered with UUID: 32c88801-2598-479f-8a94-2e3c3cdca30f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.83493492Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:51.834960833Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835681156Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835713763Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"32c88801-2598-479f-8a94-2e3c3cdca30f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835834841Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835857116Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835886727Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835903281Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.83592266Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.835947922Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836062998Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836100494Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836129534Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836157942Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"ds_connection connected from Ok(127.0.0.1:59012)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836234959Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"ds_connection connected from Ok(127.0.0.1:39029)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836488358Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"ds_connection connected from Ok(127.0.0.1:48326)","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.836542499Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.837517493Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.837583899Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.837604962Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.838201974Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.838455092Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.838741157Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.838780227Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839273526Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[0]R flush_numbers[0..12]: [346, 348, 348, 339, 346, 335, 345, 338, 347, 347, 345, 348]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839444105Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[0]R generation[0..12]: [192, 194, 194, 187, 192, 184, 191, 186, 193, 193, 191, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839468746Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839486692Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[1]R flush_numbers[0..12]: [346, 348, 348, 339, 346, 335, 345, 338, 347, 347, 345, 348]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839504098Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[1]R generation[0..12]: [192, 194, 194, 187, 192, 184, 191, 186, 193, 193, 191, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839520872Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839538258Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[2]R flush_numbers[0..12]: [346, 348, 348, 339, 346, 335, 345, 338, 347, 347, 345, 348]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839555213Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[2]R generation[0..12]: [192, 194, 194, 187, 192, 184, 191, 186, 193, 193, 191, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839571596Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.83958836Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Max found gen is 195","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839604193Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Generation requested: 198 >= found:195","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839620326Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839635537Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839650628Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"":"downstairs","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.83966643Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839684828Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"0","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-23T06:11:51.839701562Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"1","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839726844Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"client":"2","":"downstairs","session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839743257Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839757637Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"32c88801-2598-479f-8a94-2e3c3cdca30f is now active with session: bf13df08-dccc-4ada-b01f-a4f984713413","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839772788Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839786866Z","hostname":"bmat-EVT22200007-0000589c","pid":1979,"session_id":"bf13df08-dccc-4ada-b01f-a4f984713413"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839801466Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} {"msg":"Activated sub_volume 32c88801-2598-479f-8a94-2e3c3cdca30f","v":0,"name":"crucible","level":30,"time":"2025-03-23T06:11:51.839815255Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 36864 data: 10 10 8 8 11 11 8 8 8 02/30 Read block 523 len 40960 03/30 Write block 525 len 4096 data: 11 04/30 Read block 514 len 20480 05/30 Read block 524 len 36864 06/30 Write block 411 len 24576 data: 21 19 17 16 14 9 07/30 Flush 08/30 Write block 554 len 4096 data: 20 09/30 Read block 277 len 12288 10/30 Write block 231 len 24576 data: 16 17 17 13 14 16 11/30 Read block 564 len 12288 12/30 Read block 141 len 20480 13/30 Read block 558 len 4096 14/30 Read block 43 len 8192 15/30 Write block 57 len 28672 data: 20 18 17 20 18 16 15 16/30 Read block 501 len 24576 17/30 Write block 547 len 20480 data: 15 16 13 12 11 18/30 Flush 19/30 Write block 309 len 28672 data: 9 12 12 9 9 9 9 20/30 Read block 499 len 20480 21/30 Read block 404 len 28672 22/30 Read block 352 len 16384 23/30 Read block 271 len 12288 24/30 Write block 475 len 12288 data: 8 13 13 25/30 Write block 334 len 24576 data: 19 16 16 16 15 16 26/30 Write block 313 len 12288 data: 10 10 10 27/30 Read block 371 len 4096 28/30 Write block 66 len 36864 data: 14 15 13 11 10 9 10 9 10 29/30 Write block 460 len 20480 data: 11 14 14 16 18 30/30 Read block 107 len 40960 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 10 11 12 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-23T06:11:51.866626078Z","hostname":"bmat-EVT22200007-0000589c","pid":1979} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 194 194 194 348 348 348 F F F 2 040-059 198 198 198 350 350 350 F F F 3 060-079 198 198 198 350 350 350 T T T 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 193 193 193 347 347 347 F F F 9 180-199 193 193 193 347 347 347 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 198 198 198 350 350 350 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 198 198 198 349 349 349 T T T 16 320-339 194 194 194 348 348 348 T T T 17 340-359 194 194 194 348 348 348 F F F 18 360-379 194 194 194 348 348 348 F F F 19 380-399 193 193 193 347 347 347 F F F 20 400-419 198 198 198 349 349 349 F F F 21 420-439 194 194 194 348 348 348 F F F 22 440-459 193 193 193 347 347 347 F F F 23 460-479 191 191 191 345 345 345 T T T 24 480-499 193 193 193 347 347 347 F F F 25 500-519 193 193 193 347 347 347 F F F 26 520-539 198 198 198 349 349 349 F F F 27 540-559 198 198 198 350 350 350 F F F 28 560-579 194 194 194 348 348 348 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 198, Max flush: 350 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 346 346 346 F F F 1 020-039 194 194 194 348 348 348 F F F 2 040-059 198 198 198 350 350 350 F F F 3 060-079 199 199 199 351 351 351 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 184 184 184 335 335 335 F F F 6 120-139 191 191 191 345 345 345 F F F 7 140-159 186 186 186 338 338 338 F F F 8 160-179 193 193 193 347 347 347 F F F 9 180-199 193 193 193 347 347 347 F F F 10 200-219 191 191 191 345 345 345 F F F 11 220-239 198 198 198 350 350 350 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 192 192 192 346 346 346 F F F 14 280-299 192 192 192 346 346 346 F F F 15 300-319 199 199 199 351 351 351 F F F 16 320-339 199 199 199 351 351 351 F F F 17 340-359 194 194 194 348 348 348 F F F 18 360-379 194 194 194 348 348 348 F F F 19 380-399 193 193 193 347 347 347 F F F 20 400-419 198 198 198 349 349 349 F F F 21 420-439 194 194 194 348 348 348 F F F 22 440-459 193 193 193 347 347 347 F F F 23 460-479 199 199 199 351 351 351 F F F 24 480-499 193 193 193 347 347 347 F F F 25 500-519 193 193 193 347 347 347 F F F 26 520-539 198 198 198 349 349 349 F F F 27 540-559 198 198 198 350 350 350 F F F 28 560-579 194 194 194 348 348 348 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 199, Max flush: 351 9:27 Test duration Test completed real 9:26.243296780 user 58.509812245 sys 29.102142465 trap 0.106396040 tflt 0.212403217 dflt 0.313852805 kflt 0.017108723 lock 15:03:21.170802779 slp 45:51.085276453 lat 12.632107464 stop 0.073770494