Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.302296 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020147787 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020630185 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.853794793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:23.914766905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.915586495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.96204395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.96346868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.963586935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.963606943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.963632612Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"Crucible 9d2de733-1259-43c0-bd03-f3d60412eda8 has session id: 741525d0-f245-4736-9ae8-b8d090ec291c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.964462851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Upstairs opts: Upstairs UUID: 9d2de733-1259-43c0-bd03-f3d60412eda8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.964574806Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Crucible stats registered with UUID: 9d2de733-1259-43c0-bd03-f3d60412eda8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:23.964611674Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005450666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005503763Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"9d2de733-1259-43c0-bd03-f3d60412eda8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005545091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.00556722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005649116Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005676765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005698984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005719643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005740602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005760841Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.00578276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.005803919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44743)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.006308124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45995)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.006349442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63947)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.0063755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008002591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008033649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008069907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008085287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008099166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.008112965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009105326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009137665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009154494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009856429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009896487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009924726Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009939815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009954264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009967754Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009980383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.009994383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010167414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010187963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010202572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010292218Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010312367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010327046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010340825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010354335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010368384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010381443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010394513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010407682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010558805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"downstairs","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010578244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010590813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010606812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010620672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"client":"2","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010634181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.01064655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"9d2de733-1259-43c0-bd03-f3d60412eda8 is now active with session: 741525d0-f245-4736-9ae8-b8d090ec291c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.01065955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010671719Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:24.010684059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 256 Done Done Done false 1005 Acked Read 88 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100634455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188} {"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.10073441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100778438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100805496Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100836335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.100859154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100883083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.100906011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.10093045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"741525d0-f245-4736-9ae8-b8d090ec291c"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.134647042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.179537605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.179914657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.213444268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214062018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214098276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214113095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214128424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} {"msg":"Crucible 1866eb94-8149-48f2-9ffa-fbb1b178fac7 has session id: 25a9a3ff-4099-428a-9193-efe930c25f51","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214530424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Upstairs opts: Upstairs UUID: 1866eb94-8149-48f2-9ffa-fbb1b178fac7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214582042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Crucible stats registered with UUID: 1866eb94-8149-48f2-9ffa-fbb1b178fac7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.214608691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.2457428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245780408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} {"msg":"1866eb94-8149-48f2-9ffa-fbb1b178fac7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245847605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245868174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245891933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245906722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245920641Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24593422Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24594789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245961379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245975748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.245989778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_connection connected from Ok(127.0.0.1:32929)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246068604Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_connection connected from Ok(127.0.0.1:45848)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246281113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_connection connected from Ok(127.0.0.1:52375)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246328381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246791718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246822157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246844856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246867934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246894753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.246923752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.247482814Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.247526522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24756158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248079795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248121123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248151431Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248218658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248247637Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248266286Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248279605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248293484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248307124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248320783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248334912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248348692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248362241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24837618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24838978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248403199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248417058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248429998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248443197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51Wait for a query_work_queue command to finish before sending IO "} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248466546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248479645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"":"downstairs","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248492994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248520173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"0","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248534932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"1","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248548172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"client":"2","":"downstairs","session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248561541Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248573771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"1866eb94-8149-48f2-9ffa-fbb1b178fac7 is now active with session: 25a9a3ff-4099-428a-9193-efe930c25f51","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.24858656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248598519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190,"session_id":"25a9a3ff-4099-428a-9193-efe930c25f51"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:28.248610589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 24576 data: 2 2 2 2 2 2 02/30 Read block 462 len 28672 03/30 Flush 04/30 Write block 539 len 16384 data: 2 2 2 2 05/30 Write block 393 len 8192 data: 2 2 06/30 Write block 556 len 4096 data: 2 07/30 Write block 514 len 28672 data: 2 2 2 2 2 2 2 08/30 Write block 68 len 12288 data: 2 2 2 09/30 Flush 10/30 Write block 583 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Read block 48 len 16384 12/30 Flush 13/30 Write block 90 len 28672 data: 2 2 2 2 2 2 2 14/30 Flush 15/30 Write block 526 len 8192 data: 2 2 16/30 Write block 290 len 4096 data: 2 17/30 Read block 161 len 12288 18/30 Read block 113 len 4096 19/30 Read block 151 len 28672 20/30 Read block 76 len 28672 21/30 Read block 81 len 40960 22/30 Write block 260 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Write block 196 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Read block 39 len 28672 25/30 Read block 169 len 32768 26/30 Read block 418 len 12288 27/30 Write block 62 len 12288 data: 2 2 2 28/30 Write block 559 len 28672 data: 2 2 2 2 2 2 2 29/30 Read block 131 len 8192 30/30 Read block 308 len 16384 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:28.268198666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 3 3 3 T T T 4 080-099 2 2 2 5 5 5 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 T T T 27 540-559 2 2 2 3 3 3 T T T 28 560-579 1 1 1 1 1 1 T T T 29 580-599 2 2 2 4 4 4 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 2 2 2 5 5 5 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 3 3 3 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 3 3 3 6 6 6 F F F 14 280-299 3 3 3 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 3 3 3 6 6 6 F F F 28 560-579 3 3 3 6 6 6 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.590974133Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.635432357Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636044177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636080525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636103454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636122553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} {"msg":"Crucible 953debfe-1e7a-4293-84b0-bd2d7e1f0c92 has session id: adf86670-d405-49c8-bc8f-d63a36e9a6b0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636567891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Upstairs opts: Upstairs UUID: 953debfe-1e7a-4293-84b0-bd2d7e1f0c92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636613879Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Crucible stats registered with UUID: 953debfe-1e7a-4293-84b0-bd2d7e1f0c92","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.636639108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.666972276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667009384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} {"msg":"953debfe-1e7a-4293-84b0-bd2d7e1f0c92 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667025964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667042633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667065322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667079771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.6670938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667107489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667121259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667134828Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667150058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667175956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39476)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.66730957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35272)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667351717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62627)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.667425674Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668235554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668276532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668304411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.66832908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668357278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668386317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.668911801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669040374Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669069693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669486832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669512151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669526801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669605417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669624456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669638015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669650335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669663564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 6, 5, 1, 1, 1, 1, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669681453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 3, 2, 1, 1, 1, 1, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669695502Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669709632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 6, 5, 1, 1, 1, 1, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669723141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 3, 2, 1, 1, 1, 1, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.66973661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669750519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 6, 5, 1, 1, 1, 1, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669764649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 3, 2, 1, 1, 1, 1, 3, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669778248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669792097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669805077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669825376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669838335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669851255Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"":"downstairs","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669864404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669878223Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"0","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669891583Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"1","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669904602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"client":"2","":"downstairs","session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669917781Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669930091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"953debfe-1e7a-4293-84b0-bd2d7e1f0c92 is now active with session: adf86670-d405-49c8-bc8f-d63a36e9a6b0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.66994271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669954769Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198,"session_id":"adf86670-d405-49c8-bc8f-d63a36e9a6b0"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:32.669971309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 332 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 529 len 16384 03/30 Write block 355 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Read block 146 len 20480 05/30 Read block 594 len 4096 06/30 Write block 488 len 28672 data: 2 2 2 2 2 2 2 07/30 Write block 127 len 16384 data: 2 2 2 2 08/30 Read block 538 len 16384 09/30 Read block 384 len 32768 10/30 Write block 433 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 158 len 8192 data: 2 2 12/30 Read block 21 len 32768 13/30 Write block 224 len 12288 data: 2 2 2 14/30 Read block 409 len 40960 15/30 Read block 266 len 40960 16/30 Read block 87 len 32768 17/30 Write block 508 len 28672 data: 2 2 2 2 2 2 3 18/30 Read block 382 len 16384 19/30 Write block 370 len 16384 data: 2 2 2 2 20/30 Read block 465 len 8192 21/30 Write block 103 len 20480 data: 2 2 2 2 2 22/30 Write block 324 len 8192 data: 2 2 23/30 Flush 24/30 Write block 136 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Read block 322 len 32768 26/30 Write block 459 len 16384 data: 2 2 2 2 27/30 Write block 84 len 8192 data: 2 2 28/30 Write block 327 len 36864 data: 2 2 2 2 2 3 3 3 3 29/30 Write block 272 len 12288 data: 2 2 2 30/30 Write block 330 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:32.68906059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 2 2 2 5 5 5 T T T 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 T T T 7 140-159 4 4 4 7 7 7 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 3 3 3 6 6 6 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 3 3 3 6 6 6 T T T 14 280-299 3 3 3 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 4 4 4 7 7 7 T T T 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 4 4 4 7 7 7 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 3 3 3 6 6 6 F F F 28 560-579 3 3 3 6 6 6 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 4, Max flush: 7 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 3 3 3 6 6 6 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 3 3 3 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 8 8 8 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 5 5 5 8 8 8 F F F 23 460-479 5 5 5 8 8 8 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 3 3 3 6 6 6 F F F 28 560-579 3 3 3 6 6 6 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:36.964649707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.011679895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012373231Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012409409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012423558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012439058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} {"msg":"Crucible 00afb537-c112-4097-bc2c-ac4965e4cb1f has session id: 64f06e70-95ce-4af2-aeb0-0240cb8aabe1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012851547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Upstairs opts: Upstairs UUID: 00afb537-c112-4097-bc2c-ac4965e4cb1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012897265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Crucible stats registered with UUID: 00afb537-c112-4097-bc2c-ac4965e4cb1f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.012921904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.042957807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.042987005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} {"msg":"00afb537-c112-4097-bc2c-ac4965e4cb1f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043035253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043073011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043114789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043149957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043179436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043209594Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043237643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043265392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.04329404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043324019Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:38611)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043354227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45378)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043383056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63133)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.043453793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044166847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044195586Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044210555Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044224365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044238964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044252583Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044875632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.04491973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.044953449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045410076Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045438785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045453744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045510831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045529411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.04554329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045556089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045569889Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 6, 8, 7, 8, 8, 1, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045583478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 3, 5, 4, 5, 5, 1, 3, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045597417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045611706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 6, 8, 7, 8, 8, 1, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045625426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 3, 5, 4, 5, 5, 1, 3, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045642215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045656794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 6, 8, 7, 8, 8, 1, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045670453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 3, 5, 4, 5, 5, 1, 3, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045684123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045698112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045711211Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045724551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.04573756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.0457506Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"":"downstairs","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045763919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:04:37.045776028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"0","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045799157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"1","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045812326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"client":"2","":"downstairs","session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045825666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045838095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"00afb537-c112-4097-bc2c-ac4965e4cb1f is now active with session: 64f06e70-95ce-4af2-aeb0-0240cb8aabe1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045852944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045866134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206,"session_id":"64f06e70-95ce-4af2-aeb0-0240cb8aabe1"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:37.045878513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 8192 data: 2 2 02/30 Write block 228 len 24576 data: 2 2 2 2 2 2 03/30 Read block 219 len 20480 04/30 Read block 322 len 20480 05/30 Write block 214 len 32768 data: 2 2 2 2 2 2 2 2 06/30 Write block 101 len 36864 data: 2 2 3 3 3 3 3 2 2 07/30 Read block 501 len 36864 08/30 Write block 235 len 32768 data: 2 2 2 2 2 2 2 3 09/30 Read block 466 len 8192 10/30 Read block 166 len 28672 11/30 Flush 12/30 Write block 414 len 12288 data: 2 2 2 13/30 Write block 96 len 40960 data: 3 2 2 2 2 3 3 4 4 4 14/30 Write block 505 len 36864 data: 2 2 2 3 3 3 3 3 3 15/30 Read block 409 len 40960 16/30 Write block 508 len 8192 data: 4 4 17/30 Write block 568 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Read block 99 len 36864 19/30 Read block 120 len 8192 20/30 Read block 141 len 16384 21/30 Read block 207 len 40960 22/30 Read block 567 len 40960 23/30 Read block 49 len 24576 24/30 Write block 366 len 20480 data: 2 2 2 2 3 25/30 Write block 6 len 20480 data: 2 2 2 2 2 26/30 Write block 119 len 12288 data: 2 2 2 27/30 Read block 268 len 12288 28/30 Read block 15 len 16384 29/30 Write block 551 len 8192 data: 2 2 30/30 Write block 65 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:37.0650559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 6 6 6 T T T 4 080-099 5 5 5 8 8 8 T T T 5 100-119 6 6 6 9 9 9 T T T 6 120-139 5 5 5 8 8 8 T T T 7 140-159 5 5 5 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 3 3 3 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 8 8 8 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 4 4 4 7 7 7 T T T 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 4 4 4 7 7 7 F F F 22 440-459 5 5 5 8 8 8 F F F 23 460-479 5 5 5 8 8 8 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 4 4 4 7 7 7 T T T 26 520-539 3 3 3 6 6 6 F F F 27 540-559 6 6 6 9 9 9 T T T 28 560-579 3 3 3 6 6 6 T T T 29 580-599 2 2 2 4 4 4 F F F Max gen: 6, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 7 7 7 10 10 10 F F F 4 080-099 7 7 7 10 10 10 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 7 7 7 10 10 10 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 3 3 3 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 8 8 8 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 10 10 10 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 5 5 5 8 8 8 F F F 23 460-479 5 5 5 8 8 8 F F F 24 480-499 4 4 4 7 7 7 F F F 25 500-519 7 7 7 10 10 10 F F F 26 520-539 3 3 3 6 6 6 F F F 27 540-559 7 7 7 10 10 10 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.389840181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.435345863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.43601015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436048229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436062478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436076777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} {"msg":"Crucible 91e59a30-1d82-4395-9445-eb71c65eace2 has session id: 5c430c62-b68f-4cdd-9679-f8bb6dfb23e1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436512786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Upstairs opts: Upstairs UUID: 91e59a30-1d82-4395-9445-eb71c65eace2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436556224Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Crucible stats registered with UUID: 91e59a30-1d82-4395-9445-eb71c65eace2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.436581532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468024756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468060564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} {"msg":"91e59a30-1d82-4395-9445-eb71c65eace2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468097783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468114752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.46814206Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.46815963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468177199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468198318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468215977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468232726Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468250225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468278824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45289)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468311082Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52160)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.46834941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61374)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.468384348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.46915693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469186689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469202798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469217247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469231127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469245046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469866366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.469998549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470042767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.47039451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470424318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470441337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470454647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470468706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470482225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470546932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470567371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[0]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.47058276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[0]R generation[0..12]: [7, 1, 1, 7, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.47059679Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470610869Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[1]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470624568Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[1]R generation[0..12]: [7, 1, 1, 7, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470637928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470654797Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[2]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470668956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[2]R generation[0..12]: [7, 1, 1, 7, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470682655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470696635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470709764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470723064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470736053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470749102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"":"downstairs","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470771221Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.47079454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"0","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470821849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"1","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470847167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"client":"2","":"downstairs","session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470864317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470877096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 91e59a30-1d82-4395-9445-eb71c65eace2 is now active with session: 5c430c62-b68f-4cdd-9679-f8bb6dfb23e1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470893325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470909544Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214,"session_id":"5c430c62-b68f-4cdd-9679-f8bb6dfb23e1"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:41.470922234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 8192 data: 2 2 02/30 Read block 513 len 20480 03/30 Read block 196 len 20480 04/30 Flush 05/30 Write block 324 len 32768 data: 3 3 2 3 3 3 4 4 06/30 Read block 168 len 12288 07/30 Write block 316 len 12288 data: 2 2 2 08/30 Read block 337 len 24576 09/30 Write block 446 len 20480 data: 2 2 2 2 2 10/30 Read block 15 len 40960 11/30 Read block 375 len 8192 12/30 Write block 547 len 12288 data: 2 2 2 13/30 Flush 14/30 Write block 574 len 40960 data: 3 3 3 3 2 2 2 2 2 3 15/30 Flush 16/30 Write block 294 len 16384 data: 2 2 2 2 17/30 Read block 58 len 16384 18/30 Read block 375 len 40960 19/30 Write block 2 len 40960 data: 2 2 2 2 3 3 3 3 3 2 20/30 Write block 64 len 8192 data: 3 3 21/30 Write block 529 len 16384 data: 2 2 2 2 22/30 Read block 136 len 36864 23/30 Read block 418 len 24576 24/30 Read block 334 len 36864 25/30 Write block 496 len 24576 data: 2 2 2 2 2 2 26/30 Read block 55 len 32768 27/30 Write block 45 len 24576 data: 2 2 2 2 2 2 28/30 Write block 275 len 12288 data: 2 2 2 29/30 Write block 62 len 16384 data: 3 3 4 4 30/30 Read block 311 len 16384 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 15 13 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:41.490507361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 10 10 10 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 7 7 7 10 10 10 T T T 4 080-099 7 7 7 10 10 10 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 7 7 7 10 10 10 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 5 5 5 8 8 8 T T T 14 280-299 8 8 8 11 11 11 T T T 15 300-319 8 8 8 12 12 12 F F F 16 320-339 8 8 8 12 12 12 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 10 10 10 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 5 5 5 8 8 8 F F F 24 480-499 4 4 4 7 7 7 T T T 25 500-519 7 7 7 10 10 10 T T T 26 520-539 3 3 3 6 6 6 T T T 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 8, Max flush: 13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 14 14 14 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 9 9 9 14 14 14 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 7 7 7 10 10 10 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 7 7 7 10 10 10 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 6 6 6 F F F 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 9 9 9 14 14 14 F F F 14 280-299 9 9 9 14 14 14 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 8 8 8 12 12 12 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 7 7 7 10 10 10 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 5 5 5 8 8 8 F F F 24 480-499 9 9 9 14 14 14 F F F 25 500-519 9 9 9 14 14 14 F F F 26 520-539 9 9 9 14 14 14 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 9, Max flush: 14 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.774056887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:45.817684711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.818184287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.850741596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.851438832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.85147619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.851494369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.851509408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} {"msg":"Crucible 214dcac9-ce10-4248-ae2d-14023b895136 has session id: 85d6e44d-d816-4855-83b6-6c61b16501b8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.851946517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Upstairs opts: Upstairs UUID: 214dcac9-ce10-4248-ae2d-14023b895136, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.851995135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Crucible stats registered with UUID: 214dcac9-ce10-4248-ae2d-14023b895136","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.852021073Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882431948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882462936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} {"msg":"214dcac9-ce10-4248-ae2d-14023b895136 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882545182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882568661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88259659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882614399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882631358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882684086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882747122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.882764151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883033718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883063277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63590)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883079886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62768)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883094355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:32987)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883108605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88341003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883483566Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883556263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883582521Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883915375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.883970112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884116155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884146354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884512356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884764193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884793102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884807991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88482081Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88483558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.884848899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885047379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885067358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[0]R flush_numbers[0..12]: [14, 1, 14, 14, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885085417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[0]R generation[0..12]: [9, 1, 9, 9, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885099587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885113886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[1]R flush_numbers[0..12]: [14, 1, 14, 14, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885127535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[1]R generation[0..12]: [9, 1, 9, 9, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885140975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885154924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[2]R flush_numbers[0..12]: [14, 1, 14, 14, 10, 10, 10, 8, 1, 6, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885172143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[2]R generation[0..12]: [9, 1, 9, 9, 7, 7, 7, 5, 1, 3, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885186163Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885200112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885213371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88522671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.88524081Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885253879Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"":"downstairs","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:04:45.885267239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885286758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"0","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885300117Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"1","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885313016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"client":"2","":"downstairs","session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885326185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885338555Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"214dcac9-ce10-4248-ae2d-14023b895136 is now active with session: 85d6e44d-d816-4855-83b6-6c61b16501b8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885351194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885362994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222,"session_id":"85d6e44d-d816-4855-83b6-6c61b16501b8"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:45.885375133Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 40960 data: 3 3 3 2 2 2 2 2 2 2 02/30 Read block 27 len 36864 03/30 Flush 04/30 Write block 137 len 8192 data: 3 3 05/30 Write block 511 len 40960 data: 4 4 4 4 3 3 3 3 3 3 06/30 Read block 584 len 28672 07/30 Write block 40 len 20480 data: 2 2 2 2 2 08/30 Read block 2 len 28672 09/30 Read block 399 len 20480 10/30 Flush 11/30 Write block 324 len 40960 data: 4 4 3 4 4 4 5 5 4 4 12/30 Write block 3 len 20480 data: 3 3 3 4 4 13/30 Write block 168 len 40960 data: 2 2 2 2 2 2 2 2 2 2 14/30 Write block 164 len 8192 data: 2 2 15/30 Flush 16/30 Write block 129 len 12288 data: 3 3 2 17/30 Write block 384 len 40960 data: 2 2 2 2 2 2 2 2 2 3 18/30 Read block 65 len 4096 19/30 Write block 184 len 8192 data: 2 2 20/30 Write block 88 len 24576 data: 2 2 3 3 3 3 21/30 Read block 533 len 20480 22/30 Read block 564 len 8192 23/30 Write block 118 len 40960 data: 2 3 3 3 2 2 2 2 2 3 24/30 Write block 491 len 40960 data: 3 3 3 3 2 3 3 3 3 3 25/30 Write block 467 len 4096 data: 2 26/30 Read block 372 len 28672 27/30 Write block 297 len 40960 data: 3 2 2 2 2 2 2 2 2 2 28/30 Write block 260 len 32768 data: 3 3 3 3 3 3 3 3 29/30 Write block 306 len 40960 data: 3 2 2 2 2 2 2 2 2 2 30/30 Read block 190 len 40960 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 15 13 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:45.90476189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 7 7 7 10 10 10 T T T 5 100-119 7 7 7 10 10 10 T T T 6 120-139 10 10 10 16 16 16 T T T 7 140-159 5 5 5 8 8 8 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 3 3 3 6 6 6 T T T 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 9 9 9 14 14 14 T T T 14 280-299 9 9 9 14 14 14 T T T 15 300-319 8 8 8 12 12 12 T T T 16 320-339 10 10 10 17 17 17 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 10 10 10 15 15 15 F F F 19 380-399 10 10 10 15 15 15 T T T 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 5 5 5 8 8 8 T T T 24 480-499 9 9 9 14 14 14 T T T 25 500-519 10 10 10 16 16 16 T T T 26 520-539 10 10 10 16 16 16 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 10, Max flush: 17 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 11 11 11 18 18 18 F F F 5 100-119 11 11 11 18 18 18 F F F 6 120-139 11 11 11 18 18 18 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 6 6 6 9 9 9 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 6 6 6 9 9 9 F F F 13 260-279 11 11 11 18 18 18 F F F 14 280-299 11 11 11 18 18 18 F F F 15 300-319 11 11 11 18 18 18 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 10 10 10 15 15 15 F F F 19 380-399 11 11 11 18 18 18 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 10 10 10 16 16 16 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 11, Max flush: 18 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.245171931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.291138411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.291852426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.291888264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.291906503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.291921522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} {"msg":"Crucible 99d106a0-0eb6-4445-b9c0-91198025e404 has session id: ab973f56-175f-4b0f-946a-3dff22d6d615","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.292350101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Upstairs opts: Upstairs UUID: 99d106a0-0eb6-4445-b9c0-91198025e404, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.292396479Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Crucible stats registered with UUID: 99d106a0-0eb6-4445-b9c0-91198025e404","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.292421128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32368791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323723948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} {"msg":"99d106a0-0eb6-4445-b9c0-91198025e404 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323759227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323778376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323806344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323833513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323865901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32390003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323935748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.323970356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324012794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324049552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_connection connected from Ok(127.0.0.1:39699)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324125309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_connection connected from Ok(127.0.0.1:47392)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324207055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_connection connected from Ok(127.0.0.1:39959)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324238863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324747868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324777567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324895691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32491633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.324981967Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325012305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325413365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325456753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32552823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325909531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325954259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.325984107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326014376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326044924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326073633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32613424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326163508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[0]R flush_numbers[0..12]: [17, 1, 16, 14, 18, 18, 18, 8, 17, 18, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326179148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[0]R generation[0..12]: [10, 1, 10, 9, 11, 11, 11, 5, 10, 11, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326193387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326207846Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[1]R flush_numbers[0..12]: [17, 1, 16, 14, 18, 18, 18, 8, 17, 18, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326221556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[1]R generation[0..12]: [10, 1, 10, 9, 11, 11, 11, 5, 10, 11, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326235005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326248734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[2]R flush_numbers[0..12]: [17, 1, 16, 14, 18, 18, 18, 8, 17, 18, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326262274Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[2]R generation[0..12]: [10, 1, 10, 9, 11, 11, 11, 5, 10, 11, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326275713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326293962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326307521Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326320831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.32633462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326355569Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"":"downstairs","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326368849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326380908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"0","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326394027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"1","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326407127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"client":"2","":"downstairs","session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326420606Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326433005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"99d106a0-0eb6-4445-b9c0-91198025e404 is now active with session: ab973f56-175f-4b0f-946a-3dff22d6d615","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326445985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326457984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230,"session_id":"ab973f56-175f-4b0f-946a-3dff22d6d615"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:50.326470273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 20480 data: 2 2 2 2 2 02/30 Write block 226 len 8192 data: 3 2 03/30 Write block 294 len 24576 data: 3 3 3 4 3 3 04/30 Write block 213 len 36864 data: 2 3 3 3 3 3 3 3 3 05/30 Write block 108 len 20480 data: 3 3 2 2 2 06/30 Read block 525 len 8192 07/30 Read block 315 len 40960 08/30 Flush 09/30 Write block 480 len 12288 data: 2 2 2 10/30 Write block 235 len 4096 data: 3 11/30 Read block 499 len 12288 12/30 Read block 296 len 4096 13/30 Write block 313 len 20480 data: 3 3 3 3 3 14/30 Read block 357 len 36864 15/30 Write block 532 len 8192 data: 3 2 16/30 Read block 363 len 28672 17/30 Read block 101 len 20480 18/30 Read block 425 len 16384 19/30 Flush 20/30 Write block 338 len 32768 data: 3 3 3 2 2 2 2 3 21/30 Read block 31 len 20480 22/30 Write block 97 len 40960 data: 3 3 3 3 4 4 5 5 5 4 23/30 Write block 211 len 24576 data: 2 2 3 4 4 4 24/30 Write block 287 len 12288 data: 2 2 2 25/30 Read block 354 len 40960 26/30 Read block 588 len 24576 27/30 Read block 246 len 32768 28/30 Write block 246 len 20480 data: 3 3 2 2 2 29/30 Read block 458 len 4096 30/30 Write block 426 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:50.344827571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 11 11 11 18 18 18 T T T 5 100-119 12 12 12 19 19 19 T T T 6 120-139 11 11 11 18 18 18 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 12 12 12 19 19 19 T T T 11 220-239 12 12 12 20 20 20 F F F 12 240-259 6 6 6 9 9 9 T T T 13 260-279 11 11 11 18 18 18 F F F 14 280-299 12 12 12 19 19 19 T T T 15 300-319 12 12 12 20 20 20 F F F 16 320-339 10 10 10 17 17 17 T T T 17 340-359 12 12 12 19 19 19 T T T 18 360-379 10 10 10 15 15 15 F F F 19 380-399 11 11 11 18 18 18 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 4 4 4 7 7 7 T T T 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 12, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 10 10 10 16 16 16 F F F 3 060-079 9 9 9 14 14 14 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 13 13 13 21 21 21 F F F 6 120-139 11 11 11 18 18 18 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 12 12 12 20 20 20 F F F 12 240-259 13 13 13 21 21 21 F F F 13 260-279 11 11 11 18 18 18 F F F 14 280-299 13 13 13 21 21 21 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 10 10 10 15 15 15 F F F 19 380-399 11 11 11 18 18 18 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.62626939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.671394542Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672056939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672114546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672139055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672163154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} {"msg":"Crucible 6a691716-b882-4058-a736-07b3f3d3358b has session id: 16a96006-9787-4f7c-838f-798d98ce304e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.67264216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Upstairs opts: Upstairs UUID: 6a691716-b882-4058-a736-07b3f3d3358b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672685218Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Crucible stats registered with UUID: 6a691716-b882-4058-a736-07b3f3d3358b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.672706467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703054225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703090383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} {"msg":"6a691716-b882-4058-a736-07b3f3d3358b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70314983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703176429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703201678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703216287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703241086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703268875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703296923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703324492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70335368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703382059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52087)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703411677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62450)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.703437626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49530)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70356499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.7041767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704219618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704252816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704282545Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704308963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704325383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.704829578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70497692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705027628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705334873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705361241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705375891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705468006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705493275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705523633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705662557Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705683016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[0]R flush_numbers[0..12]: [17, 1, 16, 14, 21, 21, 18, 8, 17, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705698075Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[0]R generation[0..12]: [10, 1, 10, 9, 13, 13, 11, 5, 10, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705712164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705726554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[1]R flush_numbers[0..12]: [17, 1, 16, 14, 21, 21, 18, 8, 17, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705740443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[1]R generation[0..12]: [10, 1, 10, 9, 13, 13, 11, 5, 10, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705754002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705768201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[2]R flush_numbers[0..12]: [17, 1, 16, 14, 21, 21, 18, 8, 17, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705781761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[2]R generation[0..12]: [10, 1, 10, 9, 13, 13, 11, 5, 10, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70579526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.70580919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705821989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705835188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705850697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705863907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"":"downstairs","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705877076Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705903955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"0","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705930034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"1","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705953792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"client":"2","":"downstairs","session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.705978041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.706023589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"6a691716-b882-4058-a736-07b3f3d3358b is now active with session: 16a96006-9787-4f7c-838f-798d98ce304e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.706041128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.706053567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238,"session_id":"16a96006-9787-4f7c-838f-798d98ce304e"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:54.706065917Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 24576 data: 2 2 2 2 2 2 02/30 Read block 377 len 8192 03/30 Read block 468 len 28672 04/30 Write block 365 len 8192 data: 2 3 05/30 Write block 437 len 12288 data: 3 3 3 06/30 Read block 274 len 8192 07/30 Write block 385 len 32768 data: 3 3 3 3 3 3 3 3 08/30 Write block 310 len 28672 data: 3 3 3 4 4 4 4 09/30 Read block 96 len 28672 10/30 Write block 518 len 4096 data: 4 11/30 Write block 313 len 8192 data: 5 5 12/30 Read block 458 len 24576 13/30 Write block 244 len 40960 data: 3 3 4 4 3 3 3 2 2 2 14/30 Read block 184 len 36864 15/30 Read block 322 len 36864 16/30 Write block 128 len 16384 data: 3 4 4 3 17/30 Read block 533 len 8192 18/30 Write block 163 len 4096 data: 2 19/30 Write block 260 len 24576 data: 4 4 4 4 4 4 20/30 Write block 542 len 28672 data: 3 2 2 2 2 3 3 21/30 Read block 375 len 36864 22/30 Flush 23/30 Write block 569 len 32768 data: 3 3 3 3 3 4 4 4 24/30 Read block 249 len 24576 25/30 Read block 99 len 12288 26/30 Read block 417 len 40960 27/30 Write block 73 len 8192 data: 2 2 28/30 Write block 307 len 36864 data: 3 3 3 4 4 4 6 6 5 29/30 Read block 399 len 24576 30/30 Write block 412 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 6 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:54.725088042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 14 14 14 22 22 22 F F F 3 060-079 9 9 9 14 14 14 T T T 4 080-099 13 13 13 21 21 21 F F F 5 100-119 13 13 13 21 21 21 F F F 6 120-139 14 14 14 22 22 22 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 12 12 12 20 20 20 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 13 13 13 21 21 21 F F F 15 300-319 14 14 14 22 22 22 T T T 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 14 14 14 22 22 22 F F F 20 400-419 7 7 7 10 10 10 T T T 21 420-439 14 14 14 22 22 22 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 8 8 8 13 13 13 T T T 29 580-599 8 8 8 13 13 13 F F F Max gen: 14, Max flush: 22 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 14 14 14 22 22 22 F F F 3 060-079 15 15 15 23 23 23 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 13 13 13 21 21 21 F F F 6 120-139 14 14 14 22 22 22 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 12 12 12 20 20 20 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 13 13 13 21 21 21 F F F 15 300-319 15 15 15 23 23 23 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 14 14 14 22 22 22 F F F 20 400-419 15 15 15 23 23 23 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 12 12 12 20 20 20 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 15 15 15 23 23 23 F F F 29 580-599 8 8 8 13 13 13 F F F Max gen: 15, Max flush: 23 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:58.992119709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:59.035857489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.03623027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.06878098Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069337513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069373181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.06940123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} {"msg":"Crucible 668a2fa7-d5d2-4eba-a8c8-553e10f0a3ee has session id: 0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069885486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Upstairs opts: Upstairs UUID: 668a2fa7-d5d2-4eba-a8c8-553e10f0a3ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069918404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Crucible stats registered with UUID: 668a2fa7-d5d2-4eba-a8c8-553e10f0a3ee","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.069933413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101029344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101059803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} {"msg":"668a2fa7-d5d2-4eba-a8c8-553e10f0a3ee active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101129919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101151998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101179107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101196476Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101213475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101230225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101246694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101263313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101280422Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101297221Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_connection connected from Ok(127.0.0.1:47478)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101335209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56048)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101392677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57214)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.101433135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102180118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102217796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102234665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102249654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102286333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102319141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102800537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102847015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.102992018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103333941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103369119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103393388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103454565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103471844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103485514Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103497753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103514262Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[0]R flush_numbers[0..12]: [17, 1, 22, 23, 21, 21, 22, 8, 22, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103528561Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[0]R generation[0..12]: [10, 1, 14, 15, 13, 13, 14, 5, 14, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103542421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.10355667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[1]R flush_numbers[0..12]: [17, 1, 22, 23, 21, 21, 22, 8, 22, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103570479Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[1]R generation[0..12]: [10, 1, 14, 15, 13, 13, 14, 5, 14, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103584149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103598008Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[2]R flush_numbers[0..12]: [17, 1, 22, 23, 21, 21, 22, 8, 22, 18, 21, 20]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103611607Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[2]R generation[0..12]: [10, 1, 14, 15, 13, 13, 14, 5, 14, 11, 13, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103625177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103638986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103652235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103665435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103678414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103691393Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"":"downstairs","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:04:59.103704553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103726162Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"0","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103739311Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"1","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103752421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"client":"2","":"downstairs","session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.10376574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103778059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"668a2fa7-d5d2-4eba-a8c8-553e10f0a3ee is now active with session: 0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103790919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103803038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246,"session_id":"0214e2bf-b70f-4fc8-bdd6-3ccc1b289dec"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:04:59.103815267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 28672 data: 4 4 4 4 3 4 4 02/30 Read block 146 len 40960 03/30 Write block 201 len 8192 data: 3 3 04/30 Read block 346 len 36864 05/30 Read block 560 len 16384 06/30 Write block 481 len 8192 data: 3 3 07/30 Flush 08/30 Write block 81 len 16384 data: 2 2 2 3 09/30 Write block 288 len 20480 data: 3 3 3 2 2 10/30 Read block 516 len 12288 11/30 Write block 215 len 32768 data: 5 5 4 4 4 4 4 2 12/30 Read block 321 len 8192 13/30 Read block 162 len 20480 14/30 Read block 188 len 8192 15/30 Read block 465 len 36864 16/30 Write block 383 len 20480 data: 2 3 4 4 4 17/30 Read block 141 len 4096 18/30 Write block 30 len 28672 data: 2 2 2 2 2 2 2 19/30 Flush 20/30 Write block 392 len 8192 data: 4 4 21/30 Read block 269 len 36864 22/30 Write block 527 len 4096 data: 3 23/30 Flush 24/30 Write block 557 len 20480 data: 3 3 3 3 3 25/30 Read block 242 len 40960 26/30 Write block 589 len 20480 data: 3 3 3 3 2 27/30 Read block 553 len 8192 28/30 Write block 531 len 20480 data: 3 4 3 2 2 29/30 Write block 405 len 20480 data: 2 2 2 2 2 30/30 Write block 109 len 32768 data: 4 3 3 3 2 2 2 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:04:59.12207275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 16 16 16 25 25 25 F F F 2 040-059 14 14 14 22 22 22 F F F 3 060-079 15 15 15 23 23 23 F F F 4 080-099 16 16 16 25 25 25 F F F 5 100-119 13 13 13 21 21 21 T T T 6 120-139 14 14 14 22 22 22 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 15 15 15 23 23 23 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 15 15 15 23 23 23 T T T 21 420-439 14 14 14 22 22 22 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 16 16 16 24 24 24 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 16 16 16 26 26 26 T T T 27 540-559 14 14 14 22 22 22 T T T 28 560-579 15 15 15 23 23 23 T T T 29 580-599 8 8 8 13 13 13 T T T Max gen: 16, Max flush: 26 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 16 16 16 25 25 25 F F F 2 040-059 14 14 14 22 22 22 F F F 3 060-079 15 15 15 23 23 23 F F F 4 080-099 16 16 16 25 25 25 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 14 14 14 22 22 22 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 14 14 14 22 22 22 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 14 14 14 22 22 22 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 15 15 15 23 23 23 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 17 17 17 27 27 27 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 8 8 8 12 12 12 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 16 16 16 24 24 24 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 17 17 17 27 27 27 F F F Max gen: 17, Max flush: 27 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.429302642Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.474165226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.474799665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.474836983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.474851693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.474866212Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} {"msg":"Crucible 0b85bcc5-c5eb-4a4e-9804-fc50611ada88 has session id: 5689dd3b-94d9-4e28-a38c-2115407bc5a3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.475284611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Upstairs opts: Upstairs UUID: 0b85bcc5-c5eb-4a4e-9804-fc50611ada88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.475327759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Crucible stats registered with UUID: 0b85bcc5-c5eb-4a4e-9804-fc50611ada88","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.475352518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505485346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505514195Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} {"msg":"0b85bcc5-c5eb-4a4e-9804-fc50611ada88 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505585401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505604781Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505627889Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505646038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505660168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505673697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505697636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505724635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505750233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505774742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33023)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505912655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43974)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505958093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50803)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.505990522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506848439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506879178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506895137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506909306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506923876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.506947495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.507523666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.507661809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.507692608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508074859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508106447Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508121427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508234141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.50826795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508292288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508305868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508319557Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[0]R flush_numbers[0..12]: [17, 25, 22, 23, 25, 27, 22, 8, 22, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508333366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[0]R generation[0..12]: [10, 16, 14, 15, 16, 17, 14, 5, 14, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508347315Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508361815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[1]R flush_numbers[0..12]: [17, 25, 22, 23, 25, 27, 22, 8, 22, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508378724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[1]R generation[0..12]: [10, 16, 14, 15, 16, 17, 14, 5, 14, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508392613Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508406563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[2]R flush_numbers[0..12]: [17, 25, 22, 23, 25, 27, 22, 8, 22, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508420182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[2]R generation[0..12]: [10, 16, 14, 15, 16, 17, 14, 5, 14, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508433961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.50845355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:05:03.508478619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508504438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508518707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508532106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"":"downstairs","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508545356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508557495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"0","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508570685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"1","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508584004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"client":"2","":"downstairs","session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508597363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508612142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"0b85bcc5-c5eb-4a4e-9804-fc50611ada88 is now active with session: 5689dd3b-94d9-4e28-a38c-2115407bc5a3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508625302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508637451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254,"session_id":"5689dd3b-94d9-4e28-a38c-2115407bc5a3"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:03.508649831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 40960 data: 5 4 2 2 2 2 3 4 4 3 02/30 Write block 166 len 40960 data: 2 2 3 3 3 3 3 3 3 3 03/30 Read block 407 len 32768 04/30 Read block 116 len 20480 05/30 Flush 06/30 Write block 352 len 16384 data: 2 2 2 3 07/30 Read block 383 len 8192 08/30 Read block 410 len 12288 09/30 Read block 40 len 36864 10/30 Write block 316 len 28672 data: 5 4 3 2 2 2 2 11/30 Write block 178 len 4096 data: 2 12/30 Read block 529 len 20480 13/30 Flush 14/30 Write block 439 len 16384 data: 4 3 2 2 15/30 Write block 417 len 40960 data: 2 2 2 2 2 2 2 2 2 3 16/30 Read block 210 len 8192 17/30 Write block 76 len 12288 data: 2 2 2 18/30 Read block 191 len 8192 19/30 Read block 575 len 32768 20/30 Write block 300 len 20480 data: 3 3 3 3 3 21/30 Write block 58 len 28672 data: 2 2 2 2 4 4 5 22/30 Read block 121 len 16384 23/30 Write block 361 len 8192 data: 3 3 24/30 Read block 506 len 20480 25/30 Write block 277 len 4096 data: 3 26/30 Write block 580 len 24576 data: 3 3 3 4 3 3 27/30 Read block 254 len 28672 28/30 Write block 151 len 20480 data: 2 2 2 2 2 29/30 Read block 435 len 28672 30/30 Read block 212 len 32768 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 17 17 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:03.528135343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 16 16 16 25 25 25 F F F 2 040-059 14 14 14 22 22 22 T T T 3 060-079 15 15 15 23 23 23 T T T 4 080-099 16 16 16 25 25 25 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 18 18 18 28 28 28 F F F 7 140-159 5 5 5 8 8 8 T T T 8 160-179 18 18 18 29 29 29 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 14 14 14 22 22 22 T T T 14 280-299 16 16 16 25 25 25 F F F 15 300-319 18 18 18 29 29 29 T T T 16 320-339 18 18 18 29 29 29 F F F 17 340-359 18 18 18 29 29 29 F F F 18 360-379 14 14 14 22 22 22 T T T 19 380-399 16 16 16 26 26 26 F F F 20 400-419 17 17 17 27 27 27 T T T 21 420-439 14 14 14 22 22 22 T T T 22 440-459 8 8 8 12 12 12 T T T 23 460-479 11 11 11 18 18 18 F F F 24 480-499 16 16 16 24 24 24 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 17 17 17 27 27 27 T T T Max gen: 18, Max flush: 29 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 16 16 16 25 25 25 F F F 2 040-059 19 19 19 30 30 30 F F F 3 060-079 19 19 19 30 30 30 F F F 4 080-099 16 16 16 25 25 25 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 18 18 18 28 28 28 F F F 7 140-159 19 19 19 30 30 30 F F F 8 160-179 18 18 18 29 29 29 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 19 19 19 30 30 30 F F F 14 280-299 16 16 16 25 25 25 F F F 15 300-319 19 19 19 30 30 30 F F F 16 320-339 18 18 18 29 29 29 F F F 17 340-359 18 18 18 29 29 29 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 30 30 30 F F F 21 420-439 19 19 19 30 30 30 F F F 22 440-459 19 19 19 30 30 30 F F F 23 460-479 11 11 11 18 18 18 F F F 24 480-499 16 16 16 24 24 24 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 17 17 17 27 27 27 F F F 29 580-599 19 19 19 30 30 30 F F F Max gen: 19, Max flush: 30 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.823513748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.868481317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869084047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869122865Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869137555Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869152004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} {"msg":"Crucible 138430d2-4fbb-417d-b568-1a90498bc64f has session id: 7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.86964216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Upstairs opts: Upstairs UUID: 138430d2-4fbb-417d-b568-1a90498bc64f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869689627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Crucible stats registered with UUID: 138430d2-4fbb-417d-b568-1a90498bc64f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.869715166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900108062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.90014254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} {"msg":"138430d2-4fbb-417d-b568-1a90498bc64f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900177578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900194138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900221716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900239565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900256955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900277634Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900295043Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900311722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900328931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900413087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54910)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900507672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61951)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900586818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41341)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.900634406Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.90115806Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901186269Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901201548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901215757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901381359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901413068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.901902803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902068166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902154111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902350792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.90238921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902414478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902687375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902717034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902731943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902744822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902758761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[0]R flush_numbers[0..12]: [17, 25, 30, 30, 25, 27, 28, 30, 29, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902800979Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[0]R generation[0..12]: [10, 16, 19, 19, 16, 17, 18, 19, 18, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902818638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902833128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[1]R flush_numbers[0..12]: [17, 25, 30, 30, 25, 27, 28, 30, 29, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902846997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[1]R generation[0..12]: [10, 16, 19, 19, 16, 17, 18, 19, 18, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902860556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902878935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[2]R flush_numbers[0..12]: [17, 25, 30, 30, 25, 27, 28, 30, 29, 18, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902893395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[2]R generation[0..12]: [10, 16, 19, 19, 16, 17, 18, 19, 18, 11, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902907184Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902921193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902934203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902947462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902968251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.902981521Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"":"downstairs","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.90299486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903007119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"0","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903020908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"1","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903034338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"client":"2","":"downstairs","session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903047957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903060417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"138430d2-4fbb-417d-b568-1a90498bc64f is now active with session: 7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903073396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903087575Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262,"session_id":"7d2515ca-dc8d-41d7-b32c-2b2ee903b5a5"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:07.903099785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 8192 data: 2 2 02/30 Read block 328 len 28672 03/30 Read block 359 len 32768 04/30 Flush 05/30 Write block 35 len 12288 data: 3 3 2 06/30 Write block 274 len 28672 data: 3 3 3 4 2 2 2 07/30 Flush 08/30 Write block 38 len 12288 data: 2 2 3 09/30 Read block 418 len 28672 10/30 Read block 5 len 12288 11/30 Read block 551 len 4096 12/30 Read block 180 len 24576 13/30 Write block 493 len 4096 data: 5 14/30 Read block 431 len 12288 15/30 Write block 35 len 40960 data: 4 4 3 3 3 4 3 3 3 3 16/30 Flush 17/30 Write block 469 len 12288 data: 2 2 2 18/30 Write block 115 len 24576 data: 3 3 2 3 4 4 19/30 Write block 565 len 28672 data: 3 2 2 3 4 4 4 20/30 Read block 344 len 4096 21/30 Write block 404 len 24576 data: 2 3 3 3 3 3 22/30 Read block 546 len 20480 23/30 Write block 446 len 4096 data: 3 24/30 Write block 229 len 36864 data: 3 3 3 3 3 2 4 3 3 25/30 Read block 207 len 16384 26/30 Read block 562 len 28672 27/30 Read block 476 len 8192 28/30 Read block 7 len 28672 29/30 Read block 173 len 28672 30/30 Read block 227 len 4096 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:07.922051283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 20 20 20 33 33 33 F F F 3 060-079 19 19 19 30 30 30 F F F 4 080-099 16 16 16 25 25 25 F F F 5 100-119 17 17 17 27 27 27 T T T 6 120-139 18 18 18 28 28 28 T T T 7 140-159 19 19 19 30 30 30 F F F 8 160-179 18 18 18 29 29 29 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 T T T 12 240-259 14 14 14 22 22 22 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 19 19 19 30 30 30 F F F 16 320-339 18 18 18 29 29 29 F F F 17 340-359 18 18 18 29 29 29 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 19 19 19 30 30 30 T T T 21 420-439 19 19 19 30 30 30 F F F 22 440-459 19 19 19 30 30 30 T T T 23 460-479 11 11 11 18 18 18 T T T 24 480-499 20 20 20 33 33 33 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 17 17 17 27 27 27 T T T 29 580-599 19 19 19 30 30 30 F F F Max gen: 20, Max flush: 33 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 20 20 20 33 33 33 F F F 3 060-079 19 19 19 30 30 30 F F F 4 080-099 16 16 16 25 25 25 F F F 5 100-119 21 21 21 34 34 34 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 19 19 19 30 30 30 F F F 8 160-179 18 18 18 29 29 29 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 21 21 21 34 34 34 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 19 19 19 30 30 30 F F F 16 320-339 18 18 18 29 29 29 F F F 17 340-359 18 18 18 29 29 29 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 16 16 16 26 26 26 F F F 20 400-419 21 21 21 34 34 34 F F F 21 420-439 19 19 19 30 30 30 F F F 22 440-459 21 21 21 34 34 34 F F F 23 460-479 21 21 21 34 34 34 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 14 14 14 22 22 22 F F F 26 520-539 17 17 17 27 27 27 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 21 21 21 34 34 34 F F F 29 580-599 19 19 19 30 30 30 F F F Max gen: 21, Max flush: 34 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.204513023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:12.248212385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.248602925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.280950575Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.281535836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.281571174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.281588093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.281603253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} {"msg":"Crucible 54a71aa8-0f0f-4939-a91c-bbac204ef133 has session id: 173360c1-8938-4139-b830-ad52d16060e1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.281974494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Upstairs opts: Upstairs UUID: 54a71aa8-0f0f-4939-a91c-bbac204ef133, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.282020222Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Crucible stats registered with UUID: 54a71aa8-0f0f-4939-a91c-bbac204ef133","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.282048171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312254785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312283254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} {"msg":"54a71aa8-0f0f-4939-a91c-bbac204ef133 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312336892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.31236911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312398489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312416698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312433597Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312450866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312480585Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312511703Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312537362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.31256411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36286)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312591519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48018)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312620468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46429)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.312679355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.31338987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.313420158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.313436158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.313450277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.313476736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.313493115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314088695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314118954Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314135273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314680456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314710325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314725524Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314738303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314752003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314765272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314777872Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314791511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[0]R flush_numbers[0..12]: [17, 33, 33, 30, 25, 34, 34, 30, 29, 18, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.31480522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[0]R generation[0..12]: [10, 20, 20, 19, 16, 21, 21, 19, 18, 11, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314819259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314833249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[1]R flush_numbers[0..12]: [17, 33, 33, 30, 25, 34, 34, 30, 29, 18, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314846888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[1]R generation[0..12]: [10, 20, 20, 19, 16, 21, 21, 19, 18, 11, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314860377Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314874207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[2]R flush_numbers[0..12]: [17, 33, 33, 30, 25, 34, 34, 30, 29, 18, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314890796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[2]R generation[0..12]: [10, 20, 20, 19, 16, 21, 21, 19, 18, 11, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314904895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314918934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314931934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314958213Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314972762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.314985891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"":"downstairs","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315002101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315023989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"0","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315038919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"1","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315052148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"client":"2","":"downstairs","session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315065327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315077707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"54a71aa8-0f0f-4939-a91c-bbac204ef133 is now active with session: 173360c1-8938-4139-b830-ad52d16060e1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315090356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:05:12.315102416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270,"session_id":"173360c1-8938-4139-b830-ad52d16060e1"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:12.315114825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 28672 data: 2 2 2 2 2 3 3 02/30 Flush 03/30 Write block 337 len 28672 data: 3 4 4 4 3 3 3 04/30 Write block 281 len 16384 data: 2 3 3 2 05/30 Read block 136 len 4096 06/30 Write block 463 len 4096 data: 2 07/30 Read block 509 len 4096 08/30 Write block 54 len 40960 data: 3 3 3 3 3 3 3 3 5 5 09/30 Read block 86 len 8192 10/30 Write block 105 len 8192 data: 6 5 11/30 Write block 68 len 40960 data: 3 3 3 2 2 3 3 2 3 3 12/30 Read block 314 len 12288 13/30 Write block 82 len 32768 data: 3 3 4 3 2 2 3 3 14/30 Write block 384 len 16384 data: 4 5 5 5 15/30 Read block 454 len 4096 16/30 Write block 107 len 20480 data: 4 4 5 4 4 17/30 Write block 509 len 4096 data: 5 18/30 Read block 363 len 24576 19/30 Read block 219 len 24576 20/30 Flush 21/30 Write block 349 len 16384 data: 3 2 2 3 22/30 Read block 5 len 12288 23/30 Read block 565 len 20480 24/30 Write block 528 len 40960 data: 2 3 3 4 5 4 3 3 2 2 25/30 Read block 114 len 16384 26/30 Write block 285 len 24576 data: 2 2 3 4 4 4 27/30 Write block 400 len 40960 data: 2 2 2 2 3 4 4 4 4 4 28/30 Read block 538 len 8192 29/30 Read block 130 len 12288 30/30 Read block 506 len 12288 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:12.334611186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 19 19 19 30 30 30 F F F 8 160-179 22 22 22 35 35 35 F F F 9 180-199 22 22 22 35 35 35 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 21 21 21 34 34 34 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 22 22 22 36 36 36 T T T 15 300-319 19 19 19 30 30 30 F F F 16 320-339 22 22 22 36 36 36 F F F 17 340-359 22 22 22 36 36 36 T T T 18 360-379 19 19 19 30 30 30 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 21 21 21 34 34 34 T T T 21 420-439 19 19 19 30 30 30 F F F 22 440-459 21 21 21 34 34 34 F F F 23 460-479 22 22 22 36 36 36 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 17 17 17 27 27 27 T T T 27 540-559 17 17 17 27 27 27 F F F 28 560-579 21 21 21 34 34 34 F F F 29 580-599 19 19 19 30 30 30 F F F Max gen: 22, Max flush: 36 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 17 17 17 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 21 21 21 34 34 34 F F F 7 140-159 19 19 19 30 30 30 F F F 8 160-179 22 22 22 35 35 35 F F F 9 180-199 22 22 22 35 35 35 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 21 21 21 34 34 34 F F F 12 240-259 14 14 14 22 22 22 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 19 19 19 30 30 30 F F F 16 320-339 22 22 22 36 36 36 F F F 17 340-359 23 23 23 37 37 37 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 19 19 19 30 30 30 F F F 22 440-459 21 21 21 34 34 34 F F F 23 460-479 22 22 22 36 36 36 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 21 21 21 34 34 34 F F F 29 580-599 19 19 19 30 30 30 F F F Max gen: 23, Max flush: 37 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.634160627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.679554475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680245301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.68027722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680291359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680306518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} {"msg":"Crucible 1ff02cf7-ab56-4718-a4bd-beb1f7c3d3bd has session id: 3bba1c1b-a44d-419f-90f6-8df214e7f0e7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680777085Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Upstairs opts: Upstairs UUID: 1ff02cf7-ab56-4718-a4bd-beb1f7c3d3bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680823853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Crucible stats registered with UUID: 1ff02cf7-ab56-4718-a4bd-beb1f7c3d3bd","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.680843082Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711030498Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711059976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} {"msg":"1ff02cf7-ab56-4718-a4bd-beb1f7c3d3bd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711126343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711144782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711171951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711197969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711225138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711252237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711278925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711305894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711336873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711365611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54293)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.71140001Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48624)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711429038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39651)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.711458816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712107745Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712138583Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712154962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712169752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712327434Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712375702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712878997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712909355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.712925084Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713461038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713489597Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713504666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713520965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713534894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713548184Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713560463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713573863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[0]R flush_numbers[0..12]: [17, 33, 36, 36, 36, 36, 34, 30, 35, 35, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713587522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[0]R generation[0..12]: [10, 20, 22, 22, 22, 22, 21, 19, 22, 22, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713601331Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713615541Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[1]R flush_numbers[0..12]: [17, 33, 36, 36, 36, 36, 34, 30, 35, 35, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.71362955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[1]R generation[0..12]: [10, 20, 22, 22, 22, 22, 21, 19, 22, 22, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713643459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713657528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[2]R flush_numbers[0..12]: [17, 33, 36, 36, 36, 36, 34, 30, 35, 35, 25, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713671338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[2]R generation[0..12]: [10, 20, 22, 22, 22, 22, 21, 19, 22, 22, 16, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713684947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713698917Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713713686Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713727165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713740284Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.71381876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"":"downstairs","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.71383499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713847579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"0","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713861598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"1","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713874988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"client":"2","":"downstairs","session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713888427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713900727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"1ff02cf7-ab56-4718-a4bd-beb1f7c3d3bd is now active with session: 3bba1c1b-a44d-419f-90f6-8df214e7f0e7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713913586Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713925725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278,"session_id":"3bba1c1b-a44d-419f-90f6-8df214e7f0e7"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:16.713937815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 269 len 4096 data: 2 02/30 Flush 03/30 Write block 35 len 12288 data: 5 5 4 04/30 Read block 282 len 4096 05/30 Write block 245 len 36864 data: 4 5 5 4 4 4 3 3 3 06/30 Write block 98 len 4096 data: 4 07/30 Read block 263 len 12288 08/30 Read block 546 len 40960 09/30 Write block 19 len 32768 data: 2 2 3 3 2 2 2 2 10/30 Write block 25 len 24576 data: 3 3 2 2 2 3 11/30 Write block 203 len 20480 data: 3 3 3 2 2 12/30 Write block 191 len 4096 data: 2 13/30 Write block 91 len 8192 data: 4 4 14/30 Write block 579 len 40960 data: 3 4 4 4 5 4 4 3 3 3 15/30 Read block 177 len 40960 16/30 Read block 197 len 4096 17/30 Read block 153 len 36864 18/30 Read block 105 len 40960 19/30 Write block 189 len 28672 data: 2 2 3 2 2 2 2 20/30 Write block 307 len 12288 data: 4 4 4 21/30 Read block 540 len 16384 22/30 Write block 172 len 36864 data: 4 4 4 4 3 3 3 3 3 23/30 Write block 138 len 28672 data: 5 4 3 3 3 3 3 24/30 Flush 25/30 Write block 322 len 12288 data: 3 2 5 26/30 Write block 574 len 8192 data: 5 5 27/30 Read block 257 len 36864 28/30 Read block 65 len 8192 29/30 Read block 427 len 16384 30/30 Write block 318 len 8192 data: 4 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:16.732288622Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 24 24 24 39 39 39 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 24 24 24 39 39 39 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 24 24 24 39 39 39 F F F 10 200-219 24 24 24 39 39 39 F F F 11 220-239 21 21 21 34 34 34 F F F 12 240-259 24 24 24 39 39 39 F F F 13 260-279 24 24 24 38 38 38 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 24 24 24 39 39 39 T T T 16 320-339 22 22 22 36 36 36 T T T 17 340-359 23 23 23 37 37 37 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 19 19 19 30 30 30 F F F 22 440-459 21 21 21 34 34 34 F F F 23 460-479 22 22 22 36 36 36 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 24 24 24 39 39 39 T T T 29 580-599 24 24 24 39 39 39 F F F Max gen: 24, Max flush: 39 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 24 24 24 39 39 39 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 24 24 24 39 39 39 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 24 24 24 39 39 39 F F F 10 200-219 24 24 24 39 39 39 F F F 11 220-239 21 21 21 34 34 34 F F F 12 240-259 24 24 24 39 39 39 F F F 13 260-279 24 24 24 38 38 38 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 25 25 25 40 40 40 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 23 23 23 37 37 37 F F F 18 360-379 19 19 19 30 30 30 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 19 19 19 30 30 30 F F F 22 440-459 21 21 21 34 34 34 F F F 23 460-479 22 22 22 36 36 36 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 17 17 17 27 27 27 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 24 24 24 39 39 39 F F F Max gen: 25, Max flush: 40 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:20.996025202Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.041711516Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.042517096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.042557565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.042572064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.042586753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} {"msg":"Crucible c5333434-8c7b-43ca-8980-a21878747a83 has session id: 732bbaf5-c405-42f0-9b5d-b7c746b328a0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.043039601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Upstairs opts: Upstairs UUID: c5333434-8c7b-43ca-8980-a21878747a83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.043089378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Crucible stats registered with UUID: c5333434-8c7b-43ca-8980-a21878747a83","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.043114687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073817927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073850696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} {"msg":"c5333434-8c7b-43ca-8980-a21878747a83 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073892724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073909123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073932382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073952741Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.073978129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074004038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074031017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074057026Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074083124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074109663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50507)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074213738Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64781)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074261675Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:37747)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.074402399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.07499061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075026698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075055287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075083055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075112044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075141852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075679446Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075725414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.075760682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076214859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076240278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076255017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076341323Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076371552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076388251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.07640144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.07641635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[0]R flush_numbers[0..12]: [39, 39, 36, 36, 39, 36, 39, 39, 39, 39, 39, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076434009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[0]R generation[0..12]: [24, 24, 22, 22, 24, 22, 24, 24, 24, 24, 24, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076448248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076462617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[1]R flush_numbers[0..12]: [39, 39, 36, 36, 39, 36, 39, 39, 39, 39, 39, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076476617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[1]R generation[0..12]: [24, 24, 22, 22, 24, 22, 24, 24, 24, 24, 24, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076490196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076504185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[2]R flush_numbers[0..12]: [39, 39, 36, 36, 39, 36, 39, 39, 39, 39, 39, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076517685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[2]R generation[0..12]: [24, 24, 22, 22, 24, 22, 24, 24, 24, 24, 24, 21]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076531274Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076545343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076558353Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076571592Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076584541Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.0766064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"":"downstairs","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.07661995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076634689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"0","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076648088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"1","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076661357Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"client":"2","":"downstairs","session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076674437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076686836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"c5333434-8c7b-43ca-8980-a21878747a83 is now active with session: 732bbaf5-c405-42f0-9b5d-b7c746b328a0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076699746Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076712145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286,"session_id":"732bbaf5-c405-42f0-9b5d-b7c746b328a0"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:21.076724425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 460 len 28672 data: 3 3 3 3 2 2 2 02/30 Read block 362 len 4096 03/30 Read block 504 len 24576 04/30 Write block 236 len 32768 data: 4 4 3 3 3 3 4 3 05/30 Write block 62 len 36864 data: 6 6 6 5 2 2 4 4 4 06/30 Write block 262 len 16384 data: 5 5 5 5 07/30 Write block 439 len 8192 data: 5 4 08/30 Read block 396 len 12288 09/30 Read block 150 len 24576 10/30 Write block 541 len 40960 data: 3 4 3 3 3 3 4 4 3 2 11/30 Flush 12/30 Write block 353 len 40960 data: 3 3 4 3 3 3 3 3 4 4 13/30 Read block 529 len 24576 14/30 Flush 15/30 Write block 280 len 8192 data: 3 3 16/30 Read block 560 len 4096 17/30 Flush 18/30 Write block 132 len 12288 data: 3 3 3 19/30 Flush 20/30 Write block 370 len 40960 data: 4 4 4 4 3 3 3 3 3 3 21/30 Read block 462 len 28672 22/30 Write block 218 len 12288 data: 5 5 5 23/30 Read block 178 len 12288 24/30 Read block 514 len 24576 25/30 Write block 307 len 8192 data: 5 5 26/30 Read block 295 len 24576 27/30 Write block 585 len 40960 data: 5 4 4 4 4 4 4 4 3 2 28/30 Write block 154 len 36864 data: 3 3 2 2 3 3 2 2 2 29/30 Write block 23 len 4096 data: 3 30/30 Read block 591 len 12288 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:21.09653042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 24 24 24 39 39 39 T T T 2 040-059 22 22 22 36 36 36 F F F 3 060-079 26 26 26 41 41 41 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 24 24 24 39 39 39 T T T 8 160-179 24 24 24 39 39 39 T T T 9 180-199 24 24 24 39 39 39 F F F 10 200-219 24 24 24 39 39 39 T T T 11 220-239 26 26 26 41 41 41 T T T 12 240-259 26 26 26 41 41 41 F F F 13 260-279 26 26 26 41 41 41 F F F 14 280-299 26 26 26 43 43 43 F F F 15 300-319 25 25 25 40 40 40 T T T 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 26 26 26 42 42 42 T T T 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 26 26 26 41 41 41 F F F 22 440-459 26 26 26 41 41 41 F F F 23 460-479 26 26 26 41 41 41 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 24 24 24 39 39 39 T T T Max gen: 26, Max flush: 44 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 27 27 27 45 45 45 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 26 26 26 41 41 41 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 22 22 22 36 36 36 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 27 27 27 45 45 45 F F F 9 180-199 24 24 24 39 39 39 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 27 27 27 45 45 45 F F F 12 240-259 26 26 26 41 41 41 F F F 13 260-279 26 26 26 41 41 41 F F F 14 280-299 26 26 26 43 43 43 F F F 15 300-319 27 27 27 45 45 45 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 26 26 26 41 41 41 F F F 22 440-459 26 26 26 41 41 41 F F F 23 460-479 26 26 26 41 41 41 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 22 22 22 36 36 36 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 27 27 27 45 45 45 F F F Max gen: 27, Max flush: 45 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.386981878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:25.430622912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.431020113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.463550993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464117836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464154504Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464169363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464183902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} {"msg":"Crucible ff8e9f06-1bdf-4005-9834-505da125c8d5 has session id: 3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.46464605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Upstairs opts: Upstairs UUID: ff8e9f06-1bdf-4005-9834-505da125c8d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464690947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Crucible stats registered with UUID: ff8e9f06-1bdf-4005-9834-505da125c8d5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.464716346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.494908172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.49494328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} {"msg":"ff8e9f06-1bdf-4005-9834-505da125c8d5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.494965409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.494983878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495011097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495036635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495069604Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495102792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495135831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495164369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495183198Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47181)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495200597Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495218636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45259)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495250305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56057)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495407687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495882894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.495930821Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.496064725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.496109152Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.49636465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.496396769Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.496468965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.496749881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497042497Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497071905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497087895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497108743Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497229517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497248967Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497262716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497571061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497594919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[0]R flush_numbers[0..12]: [39, 45, 36, 41, 39, 36, 44, 45, 45, 39, 45, 45]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497609899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[0]R generation[0..12]: [24, 27, 22, 26, 24, 22, 26, 27, 27, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497627618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497642237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[1]R flush_numbers[0..12]: [39, 45, 36, 41, 39, 36, 44, 45, 45, 39, 45, 45]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497656567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[1]R generation[0..12]: [24, 27, 22, 26, 24, 22, 26, 27, 27, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497670506Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497684395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[2]R flush_numbers[0..12]: [39, 45, 36, 41, 39, 36, 44, 45, 45, 39, 45, 45]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497697765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[2]R generation[0..12]: [24, 27, 22, 26, 24, 22, 26, 27, 27, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497711284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497725143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497738332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497751772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497764781Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:05:25.497777801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"":"downstairs","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497797329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497809499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"0","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497824788Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"1","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497838367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"client":"2","":"downstairs","session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497851827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497864346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"ff8e9f06-1bdf-4005-9834-505da125c8d5 is now active with session: 3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497877026Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497888995Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294,"session_id":"3ea29608-33b3-4df6-9dd3-d5a50fa3cc2b"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:25.497905174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 40960 data: 4 4 2 2 2 2 2 3 4 3 02/30 Read block 84 len 40960 03/30 Flush 04/30 Write block 191 len 24576 data: 4 3 3 3 3 3 05/30 Read block 519 len 8192 06/30 Read block 314 len 20480 07/30 Read block 400 len 20480 08/30 Write block 490 len 16384 data: 3 5 5 6 09/30 Write block 116 len 8192 data: 4 3 10/30 Write block 230 len 12288 data: 4 4 4 11/30 Flush 12/30 Write block 41 len 28672 data: 4 4 4 4 3 3 3 13/30 Write block 38 len 4096 data: 4 14/30 Read block 289 len 32768 15/30 Write block 244 len 4096 data: 4 16/30 Write block 37 len 28672 data: 5 5 4 5 5 5 5 17/30 Read block 129 len 28672 18/30 Write block 261 len 8192 data: 5 6 19/30 Write block 71 len 24576 data: 3 3 4 4 3 4 20/30 Read block 394 len 20480 21/30 Read block 351 len 4096 22/30 Write block 269 len 8192 data: 3 2 23/30 Write block 192 len 24576 data: 4 4 4 4 4 3 24/30 Read block 182 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 438 len 20480 data: 4 6 5 3 3 28/30 Read block 409 len 28672 29/30 Read block 152 len 16384 30/30 Read block 201 len 20480 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:25.51650247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 27 27 27 45 45 45 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 28 28 28 48 48 48 F F F 13 260-279 28 28 28 48 48 48 F F F 14 280-299 26 26 26 43 43 43 F F F 15 300-319 27 27 27 45 45 45 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 26 26 26 41 41 41 T T T 22 440-459 26 26 26 41 41 41 T T T 23 460-479 26 26 26 41 41 41 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 28 28 28 46 46 46 F F F 26 520-539 28 28 28 46 46 46 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 27 27 27 45 45 45 F F F Max gen: 28, Max flush: 48 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 39 39 39 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 27 27 27 45 45 45 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 28 28 28 48 48 48 F F F 13 260-279 28 28 28 48 48 48 F F F 14 280-299 26 26 26 43 43 43 F F F 15 300-319 27 27 27 45 45 45 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 22 22 22 36 36 36 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 29 29 29 49 49 49 F F F 23 460-479 26 26 26 41 41 41 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 28 28 28 46 46 46 F F F 26 520-539 28 28 28 46 46 46 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 27 27 27 45 45 45 F F F Max gen: 29, Max flush: 49 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.768088038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.812544452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.81320155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.813238598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.813253837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.813267947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} {"msg":"Crucible fcb3e078-f4a9-4854-8409-8857f581ab94 has session id: ca867eaa-d2f8-41fe-a96b-1496b9f5edf1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.813761272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Upstairs opts: Upstairs UUID: fcb3e078-f4a9-4854-8409-8857f581ab94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.81380932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Crucible stats registered with UUID: fcb3e078-f4a9-4854-8409-8857f581ab94","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.813835689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.84411369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844149768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} {"msg":"fcb3e078-f4a9-4854-8409-8857f581ab94 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844177057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844194006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844222725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844266503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844302381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844337899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844379057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844415835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844451643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844488652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63414)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844664873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63311)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844691702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51982)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.844709781Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845264483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845295582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845311821Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.84532638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.84533991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845353279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845877923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.845922851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846037555Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846455805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846485853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846501442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846586688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846606247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846620287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846632756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846646295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[0]R flush_numbers[0..12]: [39, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846660205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[0]R generation[0..12]: [24, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846674024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846688123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[1]R flush_numbers[0..12]: [39, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846701863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[1]R generation[0..12]: [24, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846715502Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846729571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[2]R flush_numbers[0..12]: [39, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846746301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[2]R generation[0..12]: [24, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.84676031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846775159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846788228Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846801488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846814587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846827657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"":"downstairs","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846841006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pidWait for a query_work_queue command to finish before sending IO ":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846861725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"0","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846875124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"1","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846888343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"client":"2","":"downstairs","session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846901743Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846914052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"fcb3e078-f4a9-4854-8409-8857f581ab94 is now active with session: ca867eaa-d2f8-41fe-a96b-1496b9f5edf1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846926962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.846939151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302,"session_id":"ca867eaa-d2f8-41fe-a96b-1496b9f5edf1"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:29.84695309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 4096 data: 4 02/30 Read block 169 len 24576 03/30 Read block 422 len 36864 04/30 Flush 05/30 Write block 2 len 24576 data: 3 4 4 4 5 5 06/30 Read block 497 len 12288 07/30 Write block 455 len 20480 data: 2 2 2 2 3 08/30 Write block 12 len 12288 data: 2 2 2 09/30 Read block 468 len 40960 10/30 Write block 399 len 28672 data: 2 3 3 3 3 4 5 11/30 Write block 518 len 16384 data: 5 5 5 3 12/30 Read block 280 len 12288 13/30 Read block 203 len 36864 14/30 Read block 29 len 24576 15/30 Write block 577 len 20480 data: 4 3 4 5 5 16/30 Write block 255 len 4096 data: 2 17/30 Read block 214 len 24576 18/30 Read block 474 len 8192 19/30 Write block 309 len 32768 data: 5 5 5 5 7 7 6 6 20/30 Read block 25 len 24576 21/30 Read block 90 len 28672 22/30 Read block 100 len 24576 23/30 Write block 469 len 12288 data: 3 3 3 24/30 Write block 260 len 40960 data: 5 6 7 6 6 6 4 4 3 4 25/30 Flush 26/30 Write block 287 len 4096 data: 4 27/30 Write block 407 len 40960 data: 5 5 5 2 2 3 2 3 3 3 28/30 Write block 572 len 16384 data: 4 4 6 6 29/30 Read block 403 len 32768 30/30 Read block 162 len 8192 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:29.866906399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 51 51 51 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 27 27 27 45 45 45 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 30 30 30 51 51 51 F F F 14 280-299 26 26 26 43 43 43 T T T 15 300-319 30 30 30 51 51 51 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 30 30 30 51 51 51 T T T 21 420-439 29 29 29 49 49 49 F F F 22 440-459 30 30 30 51 51 51 F F F 23 460-479 30 30 30 51 51 51 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 30 30 30 51 51 51 T T T 29 580-599 30 30 30 51 51 51 F F F Max gen: 30, Max flush: 51 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 51 51 51 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 27 27 27 45 45 45 F F F 9 180-199 28 28 28 48 48 48 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 30 30 30 51 51 51 F F F 14 280-299 31 31 31 52 52 52 F F F 15 300-319 30 30 30 51 51 51 F F F 16 320-339 25 25 25 40 40 40 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 30 30 30 51 51 51 F F F 23 460-479 30 30 30 51 51 51 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 31, Max flush: 52 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.128214771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.173905894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.174531433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.174567082Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.174581481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.17459636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} {"msg":"Crucible 834fb5ad-6e37-47e6-ac58-7e7ff8fac394 has session id: 2ee0009c-f340-49ab-8fb3-8c3583e1fd26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.175038508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Upstairs opts: Upstairs UUID: 834fb5ad-6e37-47e6-ac58-7e7ff8fac394, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.175085546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Crucible stats registered with UUID: 834fb5ad-6e37-47e6-ac58-7e7ff8fac394","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.175105715Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205677022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.20570564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} {"msg":"834fb5ad-6e37-47e6-ac58-7e7ff8fac394 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205778657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205809785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205839884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205857963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205875312Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205891971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.20591122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205941659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.205976247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206011275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_connection connected from Ok(127.0.0.1:40260)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206052533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_connection connected from Ok(127.0.0.1:55830)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206087712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_connection connected from Ok(127.0.0.1:34785)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.20612316Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206826615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206867903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206899532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.2069317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206960979Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.206987797Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.2075541Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.207587218Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.207615977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208184709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208214817Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208229856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208242436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208260095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208274174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208286413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208299593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[0]R flush_numbers[0..12]: [51, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208313322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[0]R generation[0..12]: [30, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208327031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208341001Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[1]R flush_numbers[0..12]: [51, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.20835463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[1]R generation[0..12]: [30, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208368169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208382029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[2]R flush_numbers[0..12]: [51, 48, 48, 48, 39, 47, 44, 45, 45, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208395608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[2]R generation[0..12]: [30, 28, 28, 28, 24, 28, 26, 27, 27, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208408987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208422827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:05:34.208435956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208457605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208470704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208483764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"":"downstairs","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208496923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208508962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"0","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208522132Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"1","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208535291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"client":"2","":"downstairs","session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208548301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.20856072Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"834fb5ad-6e37-47e6-ac58-7e7ff8fac394 is now active with session: 2ee0009c-f340-49ab-8fb3-8c3583e1fd26","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208573459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208585349Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310,"session_id":"2ee0009c-f340-49ab-8fb3-8c3583e1fd26"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:34.208597708Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 12288 data: 3 3 3 02/30 Flush 03/30 Write block 172 len 40960 data: 5 5 5 5 4 4 4 4 4 3 04/30 Read block 190 len 4096 05/30 Read block 485 len 28672 06/30 Write block 496 len 4096 data: 5 07/30 Read block 212 len 16384 08/30 Read block 309 len 28672 09/30 Flush 10/30 Write block 322 len 36864 data: 4 3 6 5 4 5 5 5 6 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 266 len 36864 data: 5 5 4 5 3 2 3 3 4 15/30 Read block 63 len 16384 16/30 Read block 430 len 12288 17/30 Read block 529 len 12288 18/30 Read block 155 len 20480 19/30 Read block 594 len 16384 20/30 Read block 406 len 12288 21/30 Read block 46 len 32768 22/30 Write block 455 len 8192 data: 3 3 23/30 Read block 439 len 24576 24/30 Read block 163 len 36864 25/30 Write block 0 len 28672 data: 2 2 4 5 5 5 6 26/30 Flush 27/30 Write block 203 len 20480 data: 4 4 4 3 3 28/30 Read block 510 len 24576 29/30 Write block 37 len 4096 data: 6 30/30 Read block 279 len 8192 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:34.228750287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 28 28 28 48 48 48 T T T 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 32 32 32 54 54 54 F F F 9 180-199 32 32 32 54 54 54 F F F 10 200-219 27 27 27 45 45 45 T T T 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 32 32 32 58 58 58 F F F 14 280-299 31 31 31 52 52 52 F F F 15 300-319 30 30 30 51 51 51 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 32 32 32 53 53 53 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 30 30 30 51 51 51 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 32, Max flush: 58 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 33 33 33 59 59 59 F F F 2 040-059 28 28 28 48 48 48 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 28 28 28 47 47 47 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 27 27 27 45 45 45 F F F 8 160-179 32 32 32 54 54 54 F F F 9 180-199 32 32 32 54 54 54 F F F 10 200-219 33 33 33 59 59 59 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 32 32 32 58 58 58 F F F 14 280-299 31 31 31 52 52 52 F F F 15 300-319 30 30 30 51 51 51 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 32 32 32 53 53 53 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 32 32 32 58 58 58 F F F 23 460-479 30 30 30 51 51 51 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 33, Max flush: 59 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.482473331Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.527364974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.527999452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.52803923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.52805475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.528069649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} {"msg":"Crucible 7836bc50-976c-4e4e-9e66-b4507b222769 has session id: 6252d8d3-a84d-412d-9be4-d2b665707fa9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.528473689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Upstairs opts: Upstairs UUID: 7836bc50-976c-4e4e-9e66-b4507b222769, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.528522777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Crucible stats registered with UUID: 7836bc50-976c-4e4e-9e66-b4507b222769","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.528543736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.558931051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.55896235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} {"msg":"7836bc50-976c-4e4e-9e66-b4507b222769 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.558989969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559015427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559042776Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559057545Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559071535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559085164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559100003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559123642Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559148661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.55917271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62236)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559218297Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44786)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559412228Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62675)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.559458466Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560018068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560048007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560062636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560079665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560103064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560126513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560685925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560728313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.560763832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561266427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561292805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561307815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561384861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.56140688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561420979Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561433499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561447828Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[0]R flush_numbers[0..12]: [58, 59, 48, 48, 39, 47, 44, 45, 54, 54, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561461817Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[0]R generation[0..12]: [32, 33, 28, 28, 24, 28, 26, 27, 32, 32, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561478586Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561493365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[1]R flush_numbers[0..12]: [58, 59, 48, 48, 39, 47, 44, 45, 54, 54, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561507145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[1]R generation[0..12]: [32, 33, 28, 28, 24, 28, 26, 27, 32, 32, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561520764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561534834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[2]R flush_numbers[0..12]: [58, 59, 48, 48, 39, 47, 44, 45, 54, 54, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561548543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[2]R generation[0..12]: [32, 33, 28, 28, 24, 28, 26, 27, 32, 32, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561562222Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561576761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.56160072Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561633879Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561649018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561662147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"":"downstairs","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561675616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561687826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"0","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561701345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"1","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561717134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"client":"2","":"downstairs","session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561731074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561743403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"7836bc50-976c-4e4e-9e66-b4507b222769 is now active with session: 6252d8d3-a84d-412d-9be4-d2b665707fa9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561756063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561768062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318,"session_id":"6252d8d3-a84d-412d-9be4-d2b665707fa9"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:38.561780351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 36864 data: 4 4 4 5 5 4 3 3 4 02/30 Read block 383 len 20480 03/30 Flush 04/30 Write block 284 len 36864 data: 3 3 3 5 5 5 5 3 3 05/30 Write block 143 len 20480 data: 4 4 3 2 2 06/30 Write block 191 len 4096 data: 5 07/30 Read block 452 len 16384 08/30 Write block 174 len 12288 data: 6 6 5 09/30 Read block 512 len 36864 10/30 Read block 529 len 16384 11/30 Write block 44 len 8192 data: 5 4 12/30 Read block 65 len 36864 13/30 Write block 440 len 36864 data: 6 4 4 2 2 2 4 3 3 14/30 Write block 349 len 24576 data: 4 3 3 4 4 4 15/30 Read block 586 len 12288 16/30 Read block 165 len 4096 17/30 Read block 497 len 20480 18/30 Flush 19/30 Write block 69 len 28672 data: 5 5 4 4 5 5 4 20/30 Write block 531 len 28672 data: 5 6 5 4 4 3 3 21/30 Flush 22/30 Write block 21 len 16384 data: 4 4 4 3 23/30 Flush 24/30 Write block 348 len 32768 data: 4 5 4 4 5 5 5 5 25/30 Read block 288 len 12288 26/30 Write block 459 len 24576 data: 4 4 4 4 4 3 27/30 Read block 440 len 40960 28/30 Write block 95 len 16384 data: 4 5 4 5 29/30 Write block 102 len 40960 data: 5 6 6 7 6 5 5 6 5 5 30/30 Read block 86 len 32768 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 7 5 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:38.581661584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 34 34 34 63 63 63 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 34 34 34 60 60 60 T T T 5 100-119 28 28 28 47 47 47 T T T 6 120-139 26 26 26 44 44 44 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 34 34 34 61 61 61 F F F 9 180-199 34 34 34 61 61 61 F F F 10 200-219 33 33 33 59 59 59 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 32 32 32 58 58 58 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 30 30 30 51 51 51 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 34 34 34 61 61 61 T T T 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 34 34 34 61 61 61 T T T 23 460-479 30 30 30 51 51 51 T T T 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 34 34 34 62 62 62 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 34, Max flush: 63 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 34 34 34 63 63 63 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 35 35 35 64 64 64 F F F 5 100-119 35 35 35 64 64 64 F F F 6 120-139 26 26 26 44 44 44 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 34 34 34 61 61 61 F F F 9 180-199 34 34 34 61 61 61 F F F 10 200-219 33 33 33 59 59 59 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 32 32 32 58 58 58 F F F 14 280-299 34 34 34 61 61 61 F F F 15 300-319 30 30 30 51 51 51 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 35 35 35 64 64 64 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 35 35 35 64 64 64 F F F 23 460-479 35 35 35 64 64 64 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 34 34 34 62 62 62 F F F 27 540-559 26 26 26 41 41 41 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 35, Max flush: 64 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.848679104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.893898251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.894541559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.894578867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.894593057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.894608386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} {"msg":"Crucible fe37d461-84eb-4eae-a546-53face33478b has session id: 79a8b6d4-2de3-4dec-a4a5-f45d9af79f72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.895029115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Upstairs opts: Upstairs UUID: fe37d461-84eb-4eae-a546-53face33478b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.895080723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Crucible stats registered with UUID: fe37d461-84eb-4eae-a546-53face33478b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.895106241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926312147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926344536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} {"msg":"fe37d461-84eb-4eae-a546-53face33478b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926373644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926390683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926418632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926436451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.92645671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926493278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926527476Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926548035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926580044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926614712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:61197)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926672589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:54058)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926709648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_connection connected from Ok(127.0.0.1:62304)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.926900268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.927644402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.92767423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.927691039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.927705409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.927719358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.927733437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.928350687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.928480511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.928520159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.928875291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.92890328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.928918179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929017454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929039933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929053942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.92909287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929110229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[0]R flush_numbers[0..12]: [58, 63, 61, 62, 64, 64, 44, 61, 61, 61, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929131699Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[0]R generation[0..12]: [32, 34, 34, 34, 35, 35, 26, 34, 34, 34, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929145648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929159677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[1]R flush_numbers[0..12]: [58, 63, 61, 62, 64, 64, 44, 61, 61, 61, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929173466Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[1]R generation[0..12]: [32, 34, 34, 34, 35, 35, 26, 34, 34, 34, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929187166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929204875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[2]R flush_numbers[0..12]: [58, 63, 61, 62, 64, 64, 44, 61, 61, 61, 59, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929219104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[2]R generation[0..12]: [32, 34, 34, 34, 35, 35, 26, 34, 34, 34, 33, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929232763Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929255402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929282771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.92930967Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929329799Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929363557Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"":"downstairs","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929391656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929410155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"0","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929423984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"1","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929439184Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"client":"2","":"downstairs","session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929463712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929488951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"fe37d461-84eb-4eae-a546-53face33478b is now active with session: 79a8b6d4-2de3-4dec-a4a5-f45d9af79f72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929529449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326,"session_id":"79a8b6d4-2de3-4dec-a4a5-f45d9af79f72"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:42.929549578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 123 len 36864 data: 3 3 3 3 4 4 5 6 5 03/30 Write block 87 len 20480 data: 3 5 5 5 6 04/30 Write block 457 len 20480 data: 3 3 5 5 5 05/30 Write block 545 len 8192 data: 4 4 06/30 Flush 07/30 Write block 471 len 4096 data: 4 08/30 Read block 299 len 40960 09/30 Read block 200 len 8192 10/30 Write block 318 len 24576 data: 5 4 3 3 5 4 11/30 Read block 360 len 28672 12/30 Write block 542 len 20480 data: 5 4 4 5 5 13/30 Read block 395 len 20480 14/30 Write block 318 len 12288 data: 6 5 4 15/30 Write block 442 len 16384 data: 5 3 3 3 16/30 Write block 174 len 8192 data: 7 7 17/30 Flush 18/30 Write block 134 len 8192 data: 4 3 19/30 Write block 21 len 24576 data: 5 5 5 4 4 4 20/30 Read block 68 len 32768 21/30 Write block 270 len 20480 data: 4 3 4 4 5 22/30 Read block 348 len 40960 23/30 Write block 296 len 12288 data: 4 5 4 24/30 Read block 46 len 24576 25/30 Write block 219 len 4096 data: 6 26/30 Read block 505 len 36864 27/30 Read block 553 len 4096 28/30 Read block 444 len 20480 29/30 Read block 458 len 24576 30/30 Read block 56 len 28672 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:42.947110835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 34 34 34 63 63 63 T T T 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 35 35 35 64 64 64 F F F 6 120-139 36 36 36 66 66 66 T T T 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 34 34 34 61 61 61 F F F 10 200-219 33 33 33 59 59 59 T T T 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 32 32 32 58 58 58 T T T 14 280-299 34 34 34 61 61 61 T T T 15 300-319 36 36 36 67 67 67 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 35 35 35 64 64 64 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 36 36 36 67 67 67 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 34 34 34 62 62 62 F F F 27 540-559 36 36 36 67 67 67 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 36, Max flush: 67 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 35 35 35 64 64 64 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 34 34 34 61 61 61 F F F 10 200-219 37 37 37 68 68 68 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 30 30 30 51 51 51 F F F 13 260-279 37 37 37 68 68 68 F F F 14 280-299 37 37 37 68 68 68 F F F 15 300-319 36 36 36 67 67 67 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 35 35 35 64 64 64 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 30 30 30 51 51 51 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 F F F 22 440-459 36 36 36 67 67 67 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 34 34 34 62 62 62 F F F 27 540-559 36 36 36 67 67 67 F F F 28 560-579 31 31 31 52 52 52 F F F 29 580-599 30 30 30 51 51 51 F F F Max gen: 37, Max flush: 68 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.217405252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.262253137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.262837288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.262874276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.262893106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.262907955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} {"msg":"Crucible 882e4d88-a3b2-4cb2-ad11-87be64601cff has session id: fc1d8edc-5dae-4ee8-bf60-97f15a65640f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.263316875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Upstairs opts: Upstairs UUID: 882e4d88-a3b2-4cb2-ad11-87be64601cff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.263362452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Crucible stats registered with UUID: 882e4d88-a3b2-4cb2-ad11-87be64601cff","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.263387401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293686531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.29371907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} {"msg":"882e4d88-a3b2-4cb2-ad11-87be64601cff active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293740839Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293757018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293784167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293815535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293849933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293884532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.29391945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.293981487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294030935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294177047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_connection connected from Ok(127.0.0.1:65495)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294210116Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37251)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294240644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46028)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294269783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294701351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.2947428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294774408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294805926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294836105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.294858564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295369689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295491373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295533201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295858494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295915092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.295930181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296091963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296118512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296132811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.29614538Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.29615911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[0]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 61, 68, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296173069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[0]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 34, 37, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296187018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296201068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[1]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 61, 68, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296214737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[1]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 34, 37, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296228366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296242226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[2]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 61, 68, 47]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296255795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[2]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 34, 37, 28]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296269574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296286784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296300433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296323462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296347981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.29636593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"":"downstairs","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296379609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296391908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"0","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296405048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"1","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296428537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"client":"2","":"downstairs","session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296449236Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296462245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"882e4d88-a3b2-4cb2-ad11-87be64601cff is now active with session: fc1d8edc-5dae-4ee8-bf60-97f15a65640f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296475034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296489143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334,"session_id":"fc1d8edc-5dae-4ee8-bf60-97f15a65640f"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:47.296511753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 28672 data: 4 4 4 4 3 3 3 02/30 Write block 558 len 40960 data: 5 5 5 5 4 4 4 4 3 3 03/30 Flush 04/30 Write block 446 len 4096 data: 5 05/30 Read block 507 len 8192 06/30 Write block 386 len 4096 data: 6 07/30 Read block 514 len 8192 08/30 Write block 349 len 4096 data: 6 09/30 Read block 394 len 12288 10/30 Read block 167 len 12288 11/30 Flush 12/30 Write block 284 len 8192 data: 4 4 13/30 Flush 14/30 Write block 182 len 4096 data: 3 15/30 Write block 536 len 20480 data: 4 4 2 3 3 16/30 Write block 230 len 12288 data: 5 5 5 17/30 Read block 463 len 28672 18/30 Read block 233 len 28672 19/30 Read block 233 len 12288 20/30 Read block 544 len 24576 21/30 Write block 585 len 20480 data: 6 5 5 5 5 22/30 Read block 561 len 4096 23/30 Write block 422 len 20480 data: 3 3 3 3 4 24/30 Write block 421 len 32768 data: 3 4 4 4 4 5 3 3 25/30 Write block 258 len 4096 data: 2 26/30 Read block 351 len 20480 27/30 Write block 428 len 24576 data: 4 2 2 2 2 3 28/30 Read block 339 len 16384 29/30 Read block 155 len 8192 30/30 Read block 204 len 32768 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:47.313372484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 35 35 35 64 64 64 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 34 34 34 61 61 61 T T T 10 200-219 37 37 37 68 68 68 F F F 11 220-239 28 28 28 47 47 47 T T T 12 240-259 30 30 30 51 51 51 T T T 13 260-279 37 37 37 68 68 68 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 36 36 36 67 67 67 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 38 38 38 70 70 70 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 29 29 29 49 49 49 T T T 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 34 34 34 62 62 62 T T T 27 540-559 38 38 38 69 69 69 T T T 28 560-579 38 38 38 69 69 69 F F F 29 580-599 30 30 30 51 51 51 T T T Max gen: 38, Max flush: 71 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 34 34 34 61 61 61 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 35 35 35 64 64 64 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 37 37 37 68 68 68 F F F 11 220-239 39 39 39 72 72 72 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 37 37 37 68 68 68 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 36 36 36 67 67 67 F F F 16 320-339 36 36 36 67 67 67 F F F 17 340-359 38 38 38 70 70 70 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 31 31 31 52 52 52 F F F 21 420-439 39 39 39 72 72 72 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 39 39 39 72 72 72 F F F 27 540-559 39 39 39 72 72 72 F F F 28 560-579 38 38 38 69 69 69 F F F 29 580-599 39 39 39 72 72 72 F F F Max gen: 39, Max flush: 72 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.595584766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:51.639873768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.640288178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.673223229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.673792651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.673822809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.673836548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.673855627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} {"msg":"Crucible 57081fba-3631-420b-a610-3a7fe4bfdf9b has session id: 592c97a0-3fc8-49e2-9e40-2cbf40d2c242","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.674274087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Upstairs opts: Upstairs UUID: 57081fba-3631-420b-a610-3a7fe4bfdf9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.674317855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Crucible stats registered with UUID: 57081fba-3631-420b-a610-3a7fe4bfdf9b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.674342423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705465343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705500242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} {"msg":"57081fba-3631-420b-a610-3a7fe4bfdf9b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70553403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705551859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705580138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705597907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705614986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705631875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705648894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705665303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705682333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705703272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_connection connected from Ok(127.0.0.1:43994)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705852704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_connection connected from Ok(127.0.0.1:41074)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705893552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_connection connected from Ok(127.0.0.1:42400)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.705920911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.706708662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70675839Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.706784878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.706813477Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.706842005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.706871804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70736804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.707505353Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.707544691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.707859686Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.707884964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.707902804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708000649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708035757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708051946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708079375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708094544Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[0]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 72, 68, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708152651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[0]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 39, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70817196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70818755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[1]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 72, 68, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708201409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[1]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 39, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708215238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708229357Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[2]R flush_numbers[0..12]: [58, 68, 61, 62, 66, 64, 68, 61, 67, 72, 68, 72]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708243247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[2]R generation[0..12]: [32, 37, 34, 34, 36, 35, 37, 34, 36, 39, 37, 39]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708256796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708273535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708286994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708300194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708313393Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708334372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"":"downstairs","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708347582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.70836737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"0","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708394439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"1","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708422908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"client":"2","":"downstairs","session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708450697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708477835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"57081fba-3631-420b-a610-3a7fe4bfdf9b is now active with session: 592c97a0-3fc8-49e2-9e40-2cbf40d2c242","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708504844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708534692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342,"session_id":"592c97a0-3fc8-49e2-9e40-2cbf40d2c242"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:51.708554881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 368 len 12288 data: 3 3 5 02/30 Read block 186 len 24576 03/30 Read block 341 len 20480 04/30 Read block 466 len 28672 05/30 Read block 205 len 12288 06/30 Write block 573 len 28672 data: 5 7 7 5 5 4 5 07/30 Read block 135 len 20480 08/30 Read block 53 len 16384 09/30 Write block 524 len 8192 data: 3 3 10/30 Write block 588 len 16384 data: 6 6 5 5 11/30 Write block 48 len 28672 data: 3 3 3 2 3 3 4 12/30 Write block 228 len 40960 data: 3 4 6 6 6 4 3 5 5 5 13/30 Write block 110 len 36864 data: 6 6 4 3 3 4 5 4 4 14/30 Read block 339 len 28672 15/30 Read block 146 len 8192 16/30 Read block 146 len 4096 17/30 Read block 158 len 8192 18/30 Write block 522 len 28672 data: 3 3 4 4 4 5 4 19/30 Read block 356 len 20480 20/30 Write block 333 len 16384 data: 5 4 4 3 21/30 Read block 272 len 16384 22/30 Read block 591 len 32768 23/30 Read block 200 len 40960 24/30 Write block 430 len 36864 data: 3 3 3 4 3 3 3 4 5 25/30 Read block 476 len 24576 26/30 Read block 132 len 16384 27/30 Write block 362 len 40960 data: 5 3 2 3 4 3 4 4 6 5 28/30 Write block 562 len 8192 data: 5 5 29/30 Write block 401 len 32768 data: 4 4 4 5 6 5 6 6 30/30 Read block 391 len 12288 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 29 30 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:51.728469882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 34 34 34 61 61 61 T T T 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 35 35 35 64 64 64 T T T 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 37 37 37 68 68 68 F F F 11 220-239 39 39 39 72 72 72 T T T 12 240-259 39 39 39 72 72 72 F F F 13 260-279 37 37 37 68 68 68 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 36 36 36 67 67 67 F F F 16 320-339 36 36 36 67 67 67 T T T 17 340-359 38 38 38 70 70 70 F F F 18 360-379 27 27 27 45 45 45 T T T 19 380-399 38 38 38 70 70 70 F F F 20 400-419 31 31 31 52 52 52 T T T 21 420-439 39 39 39 72 72 72 T T T 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 39 39 39 72 72 72 T T T 27 540-559 39 39 39 72 72 72 F F F 28 560-579 38 38 38 69 69 69 T T T 29 580-599 39 39 39 72 72 72 T T T Max gen: 39, Max flush: 72 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 58 58 58 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 34 34 34 62 62 62 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 F F F 8 160-179 36 36 36 67 67 67 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 37 37 37 68 68 68 F F F 11 220-239 40 40 40 73 73 73 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 37 37 37 68 68 68 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 36 36 36 67 67 67 F F F 16 320-339 40 40 40 73 73 73 F F F 17 340-359 38 38 38 70 70 70 F F F 18 360-379 40 40 40 73 73 73 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 40 40 40 73 73 73 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 39 39 39 72 72 72 F F F 28 560-579 40 40 40 73 73 73 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 40, Max flush: 73 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.054105908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:56.097494985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.097863897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.130193817Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.130776249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.130805027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.130822956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.130837856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} {"msg":"Crucible f2867093-c614-4ae4-9a5b-df7282496806 has session id: 1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.131294653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Upstairs opts: Upstairs UUID: f2867093-c614-4ae4-9a5b-df7282496806, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.131339751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Crucible stats registered with UUID: f2867093-c614-4ae4-9a5b-df7282496806","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.13136402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.16329755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163333258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} {"msg":"f2867093-c614-4ae4-9a5b-df7282496806 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163386105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163406624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163433843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163452512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163471041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.16348816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163509369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163538538Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163565697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163592845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:36559)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163836473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:57374)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.163877411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_connection connected from Ok(127.0.0.1:59517)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164034193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164556428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164585376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164604705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164630524Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164730329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.164766487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165234374Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165264863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165356588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165824035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165863424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165890762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165908151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165921961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.16593568Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165948149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165961489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[0]R flush_numbers[0..12]: [58, 68, 73, 62, 66, 73, 68, 61, 67, 72, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165975028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[0]R generation[0..12]: [32, 37, 40, 34, 36, 40, 37, 34, 36, 39, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.165990047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166004437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[1]R flush_numbers[0..12]: [58, 68, 73, 62, 66, 73, 68, 61, 67, 72, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166018266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[1]R generation[0..12]: [32, 37, 40, 34, 36, 40, 37, 34, 36, 39, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166031815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166045784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[2]R flush_numbers[0..12]: [58, 68, 73, 62, 66, 73, 68, 61, 67, 72, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166062204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[2]R generation[0..12]: [32, 37, 40, 34, 36, 40, 37, 34, 36, 39, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166075963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166089982Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Max found gen is 41","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166114191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Generation requested: 42 >= found:41","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166127331Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.1661403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166153329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"":"downstairs","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166166518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166178478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"0","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166191587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"1","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166204737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"client":"2","":"downstairs","session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166217836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166230186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"f2867093-c614-4ae4-9a5b-df7282496806 is now active with session: 1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166242935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166254944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350,"session_id":"1cce6a7a-b1bb-4a50-8b0d-3c07288c8bba"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:05:56.166267164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 8192 data: 2 3 02/30 Read block 134 len 12288 03/30 Read block 351 len 12288 04/30 Read block 133 len 8192 05/30 Write block 321 len 32768 data: 4 6 5 7 6 5 6 6 06/30 Write block 409 len 12288 data: 6 3 3 07/30 Write block 330 len 32768 data: 7 6 5 6 5 5 4 4 08/30 Read block 135 len 24576 09/30 Read block 458 len 28672 10/30 Read block 398 len 20480 11/30 Write block 161 len 8192 data: 3 3 12/30 Write block 216 len 16384 data: 6 5 6 7 13/30 Write block 9 len 12288 data: 4 4 3 14/30 Read block 391 len 8192 15/30 Flush 16/30 Write block 13 len 24576 data: 3 3 2 2 2 2 17/30 Read block 283 len 32768 18/30 Read block 415 len 8192 19/30 Write block 156 len 40960 data: 3 3 4 4 3 4 4 3 3 3 20/30 Read block 574 len 24576 21/30 Read block 428 len 32768 22/30 Read block 63 len 36864 23/30 Write block 74 len 16384 data: 6 5 5 4 24/30 Read block 195 len 28672 25/30 Read block 35 len 24576 26/30 Write block 216 len 32768 data: 7 6 7 8 6 5 3 2 27/30 Write block 270 len 36864 data: 5 4 5 5 6 4 4 5 3 28/30 Write block 233 len 28672 data: 5 4 6 6 6 4 4 29/30 Write block 409 len 4096 data: 7 30/30 Write block 300 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:05:56.184056839Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 T T T 1 020-039 37 37 37 68 68 68 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 34 34 34 62 62 62 T T T 4 080-099 36 36 36 66 66 66 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 34 34 34 61 61 61 T T T 8 160-179 42 42 42 74 74 74 T T T 9 180-199 39 39 39 72 72 72 F F F 10 200-219 42 42 42 74 74 74 T T T 11 220-239 40 40 40 73 73 73 T T T 12 240-259 39 39 39 72 72 72 F F F 13 260-279 37 37 37 68 68 68 T T T 14 280-299 38 38 38 71 71 71 F F F 15 300-319 36 36 36 67 67 67 T T T 16 320-339 42 42 42 74 74 74 F F F 17 340-359 38 38 38 70 70 70 F F F 18 360-379 40 40 40 73 73 73 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 42 42 42 74 74 74 T T T 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 39 39 39 72 72 72 F F F 28 560-579 40 40 40 73 73 73 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 42, Max flush: 74 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 F F F 1 020-039 37 37 37 68 68 68 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 43 43 43 75 75 75 F F F 4 080-099 36 36 36 66 66 66 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 43 43 43 75 75 75 F F F 8 160-179 43 43 43 75 75 75 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 43 43 43 75 75 75 F F F 11 220-239 43 43 43 75 75 75 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 43 43 43 75 75 75 F F F 16 320-339 42 42 42 74 74 74 F F F 17 340-359 38 38 38 70 70 70 F F F 18 360-379 40 40 40 73 73 73 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 43 43 43 75 75 75 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 30 30 30 51 51 51 F F F 26 520-539 40 40 40 73 73 73 F F F 27 540-559 39 39 39 72 72 72 F F F 28 560-579 40 40 40 73 73 73 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 43, Max flush: 75 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.509024639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.555775131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.556436278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.556471556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.556485656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.556505805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} {"msg":"Crucible f4bbf644-96d1-471a-aacb-c955f50a4994 has session id: c873a7c6-49ea-4b31-8dff-85e268ede0ef","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.556962372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Upstairs opts: Upstairs UUID: f4bbf644-96d1-471a-aacb-c955f50a4994, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.55699925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Crucible stats registered with UUID: f4bbf644-96d1-471a-aacb-c955f50a4994","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.557015459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588495932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.58853249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} {"msg":"f4bbf644-96d1-471a-aacb-c955f50a4994 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588594257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588622725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588648294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588663203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588677503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588691102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588710791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.58873379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588759359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588789867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:48747)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588817566Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:35860)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588845034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:37138)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.588873673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.589726131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.58975602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.589771199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.589785098Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.589798638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.589859595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.590479504Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.590617947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.590653136Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591040006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591081084Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591114273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591199709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591229187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591245056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591257976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591272215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[0]R flush_numbers[0..12]: [75, 68, 73, 75, 66, 73, 68, 75, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591286334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[0]R generation[0..12]: [43, 37, 40, 43, 36, 40, 37, 43, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591300174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591314273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[1]R flush_numbers[0..12]: [75, 68, 73, 75, 66, 73, 68, 75, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591327922Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[1]R generation[0..12]: [43, 37, 40, 43, 36, 40, 37, 43, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591341322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591355881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[2]R flush_numbers[0..12]: [75, 68, 73, 75, 66, 73, 68, 75, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.59137031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[2]R generation[0..12]: [43, 37, 40, 43, 36, 40, 37, 43, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.59138413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591401459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591415888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591429458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:06:00.591443067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591464436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"":"downstairs","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591478475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591490654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"0","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591504524Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"1","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591517723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"client":"2","":"downstairs","session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591531632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591544092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"f4bbf644-96d1-471a-aacb-c955f50a4994 is now active with session: c873a7c6-49ea-4b31-8dff-85e268ede0ef","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591557551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.591570171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358,"session_id":"c873a7c6-49ea-4b31-8dff-85e268ede0ef"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:00.59158316Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 20480 data: 3 3 5 6 6 02/30 Read block 525 len 24576 03/30 Read block 295 len 4096 04/30 Read block 352 len 20480 05/30 Read block 186 len 12288 06/30 Write block 520 len 28672 data: 6 4 4 4 5 5 5 07/30 Flush 08/30 Write block 519 len 8192 data: 6 7 09/30 Read block 122 len 12288 10/30 Read block 566 len 28672 11/30 Write block 90 len 12288 data: 6 7 6 12/30 Read block 458 len 28672 13/30 Write block 558 len 16384 data: 6 6 6 6 14/30 Read block 163 len 32768 15/30 Read block 464 len 4096 16/30 Write block 287 len 36864 data: 6 6 6 6 4 4 2 4 4 17/30 Flush 18/30 Write block 353 len 4096 data: 6 19/30 Read block 230 len 8192 20/30 Read block 524 len 20480 21/30 Write block 497 len 8192 data: 5 4 22/30 Read block 16 len 4096 23/30 Write block 155 len 20480 data: 4 4 4 5 5 24/30 Write block 286 len 12288 data: 4 7 7 25/30 Read block 372 len 4096 26/30 Read block 83 len 32768 27/30 Read block 418 len 16384 28/30 Write block 284 len 4096 data: 5 29/30 Write block 32 len 20480 data: 3 3 3 6 6 30/30 Write block 315 len 16384 data: 7 7 5 7 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:00.609609824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 F F F 1 020-039 37 37 37 68 68 68 T T T 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 43 43 43 75 75 75 T T T 8 160-179 43 43 43 75 75 75 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 43 43 43 75 75 75 F F F 11 220-239 43 43 43 75 75 75 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 44 44 44 77 77 77 T T T 15 300-319 43 43 43 75 75 75 T T T 16 320-339 42 42 42 74 74 74 F F F 17 340-359 38 38 38 70 70 70 T T T 18 360-379 40 40 40 73 73 73 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 43 43 43 75 75 75 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 32 32 32 54 54 54 T T T 25 500-519 44 44 44 77 77 77 F F F 26 520-539 44 44 44 77 77 77 F F F 27 540-559 44 44 44 77 77 77 F F F 28 560-579 44 44 44 77 77 77 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 44, Max flush: 77 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 43 43 43 75 75 75 F F F 9 180-199 39 39 39 72 72 72 F F F 10 200-219 43 43 43 75 75 75 F F F 11 220-239 43 43 43 75 75 75 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 45 45 45 78 78 78 F F F 15 300-319 45 45 45 78 78 78 F F F 16 320-339 42 42 42 74 74 74 F F F 17 340-359 45 45 45 78 78 78 F F F 18 360-379 40 40 40 73 73 73 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 43 43 43 75 75 75 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 F F F 24 480-499 45 45 45 78 78 78 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 44 44 44 77 77 77 F F F 27 540-559 44 44 44 77 77 77 F F F 28 560-579 44 44 44 77 77 77 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 45, Max flush: 78 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.888570625Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:04.932274966Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.932673366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.965091292Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.965662904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.965697203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.965711102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.965725891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} {"msg":"Crucible c8822caf-4fa3-4111-8572-1afab77237af has session id: dd5de2bf-824b-4920-ad73-a68d40975af7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.966228447Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Upstairs opts: Upstairs UUID: c8822caf-4fa3-4111-8572-1afab77237af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.966272245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Crucible stats registered with UUID: c8822caf-4fa3-4111-8572-1afab77237af","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.966290294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996632962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99666967Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} {"msg":"c8822caf-4fa3-4111-8572-1afab77237af active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996735127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996754356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996776635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996790834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996805183Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996818863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996832592Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996846171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99687178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996897469Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62888)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996926937Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46223)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996954676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54259)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.996981875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.997853142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99789481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.997926888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.997956996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.997988735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.998015944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.998513649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.998636403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.998665882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999070662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.9990992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99911396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999198055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999220574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999234754Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999289671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99930833Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[0]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999323299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[0]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999337779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999352378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[1]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999366287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[1]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999380107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999394496Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[2]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 75, 72, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999408455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[2]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 43, 39, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999422144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999436284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999449673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999465762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999479462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999492811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"":"downstairs","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"No downstairs reconciliation requiredWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.99951453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999527199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"0","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999540809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"1","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999554238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"client":"2","":"downstairs","session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999567757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999580167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"c8822caf-4fa3-4111-8572-1afab77237af is now active with session: dd5de2bf-824b-4920-ad73-a68d40975af7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999593156Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:06:04.999605655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366,"session_id":"dd5de2bf-824b-4920-ad73-a68d40975af7"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:04.999620565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 32768 data: 6 4 3 3 3 4 3 4 02/30 Write block 295 len 40960 data: 5 5 6 5 4 5 5 4 4 4 03/30 Read block 431 len 28672 04/30 Write block 414 len 12288 data: 4 4 4 05/30 Read block 166 len 20480 06/30 Read block 380 len 32768 07/30 Write block 179 len 32768 data: 5 5 4 4 3 4 4 2 08/30 Read block 331 len 24576 09/30 Write block 481 len 16384 data: 4 4 2 2 10/30 Read block 247 len 24576 11/30 Write block 369 len 4096 data: 5 12/30 Read block 154 len 32768 13/30 Read block 579 len 8192 14/30 Read block 421 len 24576 15/30 Read block 38 len 4096 16/30 Flush 17/30 Write block 572 len 28672 data: 5 6 8 8 6 6 5 18/30 Read block 392 len 12288 19/30 Write block 336 len 8192 data: 5 5 20/30 Flush 21/30 Write block 367 len 16384 data: 4 5 6 7 22/30 Read block 374 len 4096 23/30 Flush 24/30 Write block 545 len 36864 data: 6 6 5 5 4 3 3 3 2 25/30 Read block 206 len 4096 26/30 Read block 440 len 32768 27/30 Read block 103 len 20480 28/30 Write block 466 len 36864 data: 3 3 2 4 4 5 2 2 2 29/30 Read block 312 len 20480 30/30 Read block 31 len 8192 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:05.019977734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 46 46 46 79 79 79 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 43 43 43 75 75 75 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 46 46 46 79 79 79 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 45 45 45 78 78 78 F F F 18 360-379 46 46 46 81 81 81 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 36 36 36 67 67 67 T T T 24 480-499 46 46 46 79 79 79 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 44 44 44 77 77 77 F F F 27 540-559 44 44 44 77 77 77 T T T 28 560-579 46 46 46 80 80 80 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 46, Max flush: 81 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 40 40 40 73 73 73 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 46 46 46 79 79 79 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 43 43 43 75 75 75 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 46 46 46 79 79 79 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 45 45 45 78 78 78 F F F 18 360-379 46 46 46 81 81 81 F F F 19 380-399 38 38 38 70 70 70 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 38 38 38 70 70 70 F F F 23 460-479 47 47 47 82 82 82 F F F 24 480-499 46 46 46 79 79 79 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 44 44 44 77 77 77 F F F 27 540-559 47 47 47 82 82 82 F F F 28 560-579 46 46 46 80 80 80 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 47, Max flush: 82 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.275553856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:09.31920703Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.31960506Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.352047495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.352586588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.352621987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.352636186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.352650695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} {"msg":"Crucible c5258282-2fc2-4fc0-b270-f67bf1bd9ece has session id: fece363f-1e51-4756-860f-2b26e50756d2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.353151321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Upstairs opts: Upstairs UUID: c5258282-2fc2-4fc0-b270-f67bf1bd9ece, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.353197758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Crucible stats registered with UUID: c5258282-2fc2-4fc0-b270-f67bf1bd9ece","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.353224467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383625332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383652511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} {"msg":"c5258282-2fc2-4fc0-b270-f67bf1bd9ece active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383687709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383704738Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383732757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383763155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383796684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.38388239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383921437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383961955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.383998464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.38407959Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61042)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384174815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38771)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384220803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63990)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384246192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384609294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384636712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384772386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384808044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384837972Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.384866791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385208474Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385336808Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385653532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385771197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385805455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385821584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385956988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385976787Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.385990836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386137489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386157938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[0]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 79, 79, 75, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386176067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[0]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 46, 46, 43, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386190086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386204455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[1]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 79, 79, 75, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386218305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[1]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 46, 46, 43, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386232024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386246233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[2]R flush_numbers[0..12]: [75, 78, 73, 76, 77, 73, 68, 78, 79, 79, 75, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386260753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[2]R generation[0..12]: [43, 45, 40, 44, 44, 40, 37, 45, 46, 46, 43, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386274302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386288431Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.38630152Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.38631496Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386327939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:06:09.386340859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"":"downstairs","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386361128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386373257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"0","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386386916Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"1","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386400356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"client":"2","":"downstairs","session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386413915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386426274Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"c5258282-2fc2-4fc0-b270-f67bf1bd9ece is now active with session: fece363f-1e51-4756-860f-2b26e50756d2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386439114Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386451383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374,"session_id":"fece363f-1e51-4756-860f-2b26e50756d2"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:09.386463732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 111 len 8192 data: 7 5 02/30 Write block 535 len 28672 data: 5 5 5 3 4 4 4 03/30 Flush 04/30 Write block 476 len 24576 data: 2 2 2 2 3 5 05/30 Read block 471 len 28672 06/30 Read block 497 len 20480 07/30 Flush 08/30 Write block 440 len 20480 data: 7 5 6 4 4 09/30 Read block 568 len 4096 10/30 Read block 272 len 40960 11/30 Write block 193 len 32768 data: 5 5 5 5 4 3 3 3 12/30 Flush 13/30 Write block 112 len 28672 data: 6 4 4 5 6 5 5 14/30 Write block 381 len 20480 data: 2 2 3 5 6 15/30 Read block 249 len 8192 16/30 Read block 522 len 8192 17/30 Read block 182 len 12288 18/30 Write block 341 len 12288 data: 4 4 4 19/30 Read block 280 len 40960 20/30 Write block 14 len 12288 data: 4 3 3 21/30 Read block 272 len 24576 22/30 Read block 340 len 4096 23/30 Read block 401 len 20480 24/30 Read block 237 len 36864 25/30 Read block 429 len 24576 26/30 Read block 142 len 32768 27/30 Read block 374 len 32768 28/30 Write block 306 len 32768 data: 4 6 6 6 6 6 6 8 29/30 Read block 232 len 28672 30/30 Read block 262 len 8192 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:09.406825751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 75 75 75 T T T 1 020-039 45 45 45 78 78 78 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 48 48 48 83 83 83 T T T 6 120-139 37 37 37 68 68 68 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 46 46 46 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 46 46 46 79 79 79 T T T 16 320-339 46 46 46 80 80 80 F F F 17 340-359 45 45 45 78 78 78 T T T 18 360-379 46 46 46 81 81 81 F F F 19 380-399 38 38 38 70 70 70 T T T 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 48 48 48 84 84 84 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 46 46 46 80 80 80 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 48, Max flush: 85 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 40 40 40 73 73 73 F F F 3 060-079 44 44 44 76 76 76 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 49 49 49 86 86 86 F F F 6 120-139 37 37 37 68 68 68 F F F 7 140-159 45 45 45 78 78 78 F F F 8 160-179 46 46 46 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 49 49 49 86 86 86 F F F 18 360-379 46 46 46 81 81 81 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 48 48 48 84 84 84 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 48 48 48 83 83 83 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 46 46 46 80 80 80 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 49, Max flush: 86 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.707991371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.75337568Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754044937Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754093354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754116403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754145062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} {"msg":"Crucible c86b54fa-1803-4737-927a-520cad01448d has session id: 13da0302-5ee0-4a69-9574-80456e9e0c0a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754528003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Upstairs opts: Upstairs UUID: c86b54fa-1803-4737-927a-520cad01448d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.754566991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Crucible stats registered with UUID: c86b54fa-1803-4737-927a-520cad01448d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.7545831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785327378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785354926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} {"msg":"c86b54fa-1803-4737-927a-520cad01448d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785425613Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785444712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785467331Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78548191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785495589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785508949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785522268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785536737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785561456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785586905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43787)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78568126Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43856)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785724458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55602)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.785861771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786441253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786469632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786588626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786629604Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786659022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.786684301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78711176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787236054Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787257893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787656103Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787696921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78772291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787784347Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787815825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787832294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787845624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787859313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[0]R flush_numbers[0..12]: [86, 78, 73, 76, 77, 86, 68, 78, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787873483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[0]R generation[0..12]: [49, 45, 40, 44, 44, 49, 37, 45, 46, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787890492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787905201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[1]R flush_numbers[0..12]: [86, 78, 73, 76, 77, 86, 68, 78, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78791924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[1]R generation[0..12]: [49, 45, 40, 44, 44, 49, 37, 45, 46, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78793322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787947319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[2]R flush_numbers[0..12]: [86, 78, 73, 76, 77, 86, 68, 78, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787961248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[2]R generation[0..12]: [49, 45, 40, 44, 44, 49, 37, 45, 46, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787975007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.787988807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788001946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788015385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788028465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788041464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"":"downstairs","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788054763Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"Wait for a query_work_queue command to finish before sending IO 13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788076333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"0","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788091272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"1","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788104681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"client":"2","":"downstairs","session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.78811812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.7881305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"c86b54fa-1803-4737-927a-520cad01448d is now active with session: 13da0302-5ee0-4a69-9574-80456e9e0c0a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788143529Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:13.788155809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382,"session_id":"13da0302-5ee0-4a69-9574-80456e9e0c0a"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:06:13.788168148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 375 len 16384 data: 4 4 4 4 02/30 Flush 03/30 Write block 105 len 36864 data: 8 7 6 6 7 7 8 7 5 04/30 Read block 536 len 28672 05/30 Read block 426 len 20480 06/30 Read block 112 len 12288 07/30 Flush 08/30 Write block 105 len 36864 data: 9 8 7 7 8 8 9 8 6 09/30 Read block 534 len 40960 10/30 Write block 463 len 32768 data: 5 4 3 4 4 3 5 5 11/30 Write block 43 len 12288 data: 6 6 5 12/30 Read block 535 len 40960 13/30 Read block 324 len 32768 14/30 Read block 88 len 20480 15/30 Write block 566 len 36864 data: 4 4 4 5 5 5 6 7 9 16/30 Write block 130 len 32768 data: 7 6 4 4 5 4 4 5 17/30 Read block 461 len 28672 18/30 Read block 533 len 16384 19/30 Write block 138 len 24576 data: 6 5 4 4 4 5 20/30 Write block 58 len 28672 data: 4 4 4 4 7 7 7 21/30 Read block 185 len 40960 22/30 Write block 43 len 24576 data: 7 7 6 4 4 4 23/30 Read block 272 len 24576 24/30 Read block 289 len 4096 25/30 Write block 63 len 24576 data: 8 8 6 4 4 6 26/30 Flush 27/30 Write block 535 len 8192 data: 6 6 28/30 Write block 232 len 32768 data: 7 6 5 7 7 7 5 5 29/30 Write block 164 len 36864 data: 4 4 3 3 4 4 4 4 6 30/30 Read block 22 len 40960 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:13.808934397Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 50 50 50 89 89 89 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 50 50 50 89 89 89 F F F 6 120-139 50 50 50 89 89 89 F F F 7 140-159 50 50 50 89 89 89 F F F 8 160-179 46 46 46 79 79 79 T T T 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 46 46 46 79 79 79 T T T 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 49 49 49 86 86 86 F F F 18 360-379 50 50 50 87 87 87 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 48 48 48 83 83 83 T T T 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 50, Max flush: 89 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 50 50 50 89 89 89 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 50 50 50 89 89 89 F F F 6 120-139 50 50 50 89 89 89 F F F 7 140-159 50 50 50 89 89 89 F F F 8 160-179 51 51 51 90 90 90 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 51 51 51 90 90 90 F F F 12 240-259 39 39 39 72 72 72 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 49 49 49 86 86 86 F F F 18 360-379 50 50 50 87 87 87 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 44 44 44 77 77 77 F F F 26 520-539 51 51 51 90 90 90 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 51, Max flush: 90 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.068105602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.112568716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113166437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113203215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113217514Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113231773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} {"msg":"Crucible bf28f4fd-17a5-42f8-90b4-b69a54464ed7 has session id: eed251ae-ac16-4aa6-9115-3eb8c531db4e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.11371807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Upstairs opts: Upstairs UUID: bf28f4fd-17a5-42f8-90b4-b69a54464ed7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113766477Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Crucible stats registered with UUID: bf28f4fd-17a5-42f8-90b4-b69a54464ed7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.113790636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144044418Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144073767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} {"msg":"bf28f4fd-17a5-42f8-90b4-b69a54464ed7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144153483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144173602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144198621Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14421351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14422752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144245029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144259188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144272697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144286557Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144300266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54138)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144401831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41844)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14462655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65005)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.144683567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14543149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.145474988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.145507257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.145540205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.145581413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.145617261Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146176434Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146209972Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146226941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146791214Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146819382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146834371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146846841Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14686051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146874169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146886379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146899548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[0]R flush_numbers[0..12]: [86, 78, 89, 89, 77, 89, 89, 89, 90, 85, 85, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146913177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[0]R generation[0..12]: [49, 45, 50, 50, 44, 50, 50, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146926947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146940916Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[1]R flush_numbers[0..12]: [86, 78, 89, 89, 77, 89, 89, 89, 90, 85, 85, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146954615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[1]R generation[0..12]: [49, 45, 50, 50, 44, 50, 50, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146968305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.146985964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[2]R flush_numbers[0..12]: [86, 78, 89, 89, 77, 89, 89, 89, 90, 85, 85, 90]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147000013Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[2]R generation[0..12]: [49, 45, 50, 50, 44, 50, 50, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147013723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147027682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147040531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147053601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.14706647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-26T22:06:18.147088499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"":"downstairs","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147102108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147114118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"0","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147127317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"1","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147140086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"client":"2","":"downstairs","session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147153286Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147165595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"bf28f4fd-17a5-42f8-90b4-b69a54464ed7 is now active with session: eed251ae-ac16-4aa6-9115-3eb8c531db4e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147178414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147192454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390,"session_id":"eed251ae-ac16-4aa6-9115-3eb8c531db4e"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:18.147204793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 83 len 32768 data: 4 5 4 3 4 6 6 7 02/30 Read block 240 len 4096 03/30 Read block 477 len 40960 04/30 Write block 369 len 36864 data: 7 8 6 5 5 4 5 5 5 05/30 Read block 489 len 32768 06/30 Read block 481 len 16384 07/30 Read block 89 len 12288 08/30 Read block 92 len 8192 09/30 Read block 469 len 8192 10/30 Write block 504 len 8192 data: 2 3 11/30 Read block 159 len 4096 12/30 Write block 101 len 8192 data: 5 6 13/30 Flush 14/30 Write block 536 len 8192 data: 7 6 15/30 Flush 16/30 Write block 224 len 24576 data: 4 4 5 4 5 5 17/30 Flush 18/30 Write block 195 len 40960 data: 6 6 5 4 4 4 4 4 5 5 19/30 Write block 164 len 36864 data: 5 5 4 4 5 5 5 5 7 20/30 Read block 58 len 40960 21/30 Read block 361 len 24576 22/30 Write block 252 len 24576 data: 4 4 2 3 2 2 23/30 Write block 345 len 40960 data: 4 4 4 5 7 5 5 6 7 6 24/30 Flush 25/30 Write block 170 len 16384 data: 6 6 8 6 26/30 Read block 470 len 20480 27/30 Write block 107 len 24576 data: 8 8 9 9 10 9 28/30 Read block 427 len 36864 29/30 Read block 136 len 28672 30/30 Read block 387 len 28672 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:18.168553073Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 50 50 50 89 89 89 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 52 52 52 91 91 91 T T T 6 120-139 50 50 50 89 89 89 F F F 7 140-159 50 50 50 89 89 89 F F F 8 160-179 52 52 52 94 94 94 T T T 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 52 52 52 93 93 93 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 52 52 52 94 94 94 F F F 18 360-379 52 52 52 91 91 91 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 52 52 52 91 91 91 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 52, Max flush: 94 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 45 45 45 78 78 78 F F F 2 040-059 50 50 50 89 89 89 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 50 50 50 89 89 89 F F F 7 140-159 50 50 50 89 89 89 F F F 8 160-179 53 53 53 95 95 95 F F F 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 52 52 52 93 93 93 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 52 52 52 94 94 94 F F F 18 360-379 52 52 52 91 91 91 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 40 40 40 73 73 73 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 52 52 52 91 91 91 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 53, Max flush: 95 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.426988875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.472269069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473021032Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.47305752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473086938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473102628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} {"msg":"Crucible 966c471f-d63c-48e2-8ebf-ab75245cf0c5 has session id: 64539d01-0d60-44bd-84ff-6e712676b96a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473564435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Upstairs opts: Upstairs UUID: 966c471f-d63c-48e2-8ebf-ab75245cf0c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473610673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Crucible stats registered with UUID: 966c471f-d63c-48e2-8ebf-ab75245cf0c5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.473637522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504039576Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504074695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} {"msg":"966c471f-d63c-48e2-8ebf-ab75245cf0c5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504095334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504111523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504139512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504157531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50417444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504191169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504207658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504223947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504241167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504268515Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56817)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504306683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57926)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504339782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59678)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.504376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505213229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505242648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505260296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505285935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50538445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505427388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505828729Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505946283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.505969752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50641611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506449288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506465367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506531364Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506552023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506565942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506578302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506591811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[0]R flush_numbers[0..12]: [86, 78, 89, 89, 91, 95, 89, 89, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50660553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[0]R generation[0..12]: [49, 45, 50, 50, 52, 53, 50, 50, 53, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50661943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506633679Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[1]R flush_numbers[0..12]: [86, 78, 89, 89, 91, 95, 89, 89, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506647328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[1]R generation[0..12]: [49, 45, 50, 50, 52, 53, 50, 50, 53, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506660918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506674827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[2]R flush_numbers[0..12]: [86, 78, 89, 89, 91, 95, 89, 89, 95, 94, 94, 93]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506688496Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[2]R generation[0..12]: [49, 45, 50, 50, 52, 53, 50, 50, 53, 52, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506702176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506716055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506732014Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506745654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506758853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506771952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"":"downstairs","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:06:22.506785532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.5068067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"0","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.50681997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"1","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506833329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"client":"2","":"downstairs","session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506846598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506859078Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"966c471f-d63c-48e2-8ebf-ab75245cf0c5 is now active with session: 64539d01-0d60-44bd-84ff-6e712676b96a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506871787Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506883777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398,"session_id":"64539d01-0d60-44bd-84ff-6e712676b96a"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:22.506895896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 40960 data: 8 4 4 4 3 5 5 5 3 3 02/30 Read block 321 len 16384 03/30 Write block 133 len 16384 data: 5 6 5 5 04/30 Write block 225 len 8192 data: 5 6 05/30 Read block 261 len 20480 06/30 Read block 304 len 32768 07/30 Flush 08/30 Write block 141 len 28672 data: 5 5 6 5 4 3 3 09/30 Read block 446 len 16384 10/30 Read block 564 len 28672 11/30 Read block 222 len 16384 12/30 Read block 568 len 24576 13/30 Write block 34 len 16384 data: 4 7 7 7 14/30 Read block 556 len 36864 15/30 Read block 480 len 36864 16/30 Write block 233 len 20480 data: 7 6 8 8 8 17/30 Read block 239 len 36864 18/30 Read block 166 len 16384 19/30 Read block 43 len 4096 20/30 Write block 125 len 20480 data: 4 4 5 5 6 21/30 Flush 22/30 Write block 42 len 28672 data: 6 8 8 7 5 5 5 23/30 Read block 151 len 8192 24/30 Read block 560 len 12288 25/30 Read block 514 len 8192 26/30 Read block 372 len 8192 27/30 Read block 419 len 16384 28/30 Read block 525 len 40960 29/30 Read block 434 len 8192 30/30 Write block 431 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:22.526618826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 50 50 50 89 89 89 T T T 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 54 54 54 97 97 97 F F F 8 160-179 53 53 53 95 95 95 F F F 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 54 54 54 97 97 97 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 52 52 52 94 94 94 F F F 18 360-379 52 52 52 91 91 91 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 54 54 54 96 96 96 F F F 21 420-439 40 40 40 73 73 73 T T T 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 52 52 52 91 91 91 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 54, Max flush: 97 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 55 55 55 98 98 98 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 54 54 54 97 97 97 F F F 8 160-179 53 53 53 95 95 95 F F F 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 54 54 54 97 97 97 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 49 49 49 86 86 86 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 52 52 52 94 94 94 F F F 18 360-379 52 52 52 91 91 91 F F F 19 380-399 49 49 49 86 86 86 F F F 20 400-419 54 54 54 96 96 96 F F F 21 420-439 55 55 55 98 98 98 F F F 22 440-459 48 48 48 85 85 85 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 48 48 48 84 84 84 F F F 25 500-519 52 52 52 91 91 91 F F F 26 520-539 52 52 52 92 92 92 F F F 27 540-559 48 48 48 83 83 83 F F F 28 560-579 50 50 50 89 89 89 F F F 29 580-599 40 40 40 73 73 73 F F F Max gen: 55, Max flush: 98 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.779676642Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.825060371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826055872Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.82609313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826107619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826122838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} {"msg":"Crucible 38876c8b-200b-4759-aaa1-c3ce8d1ee5e8 has session id: 7f10e446-4135-458a-a83b-69f7c30eb562","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826594135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Upstairs opts: Upstairs UUID: 38876c8b-200b-4759-aaa1-c3ce8d1ee5e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826636083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Crucible stats registered with UUID: 38876c8b-200b-4759-aaa1-c3ce8d1ee5e8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.826659652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.8570076Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857041668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} {"msg":"38876c8b-200b-4759-aaa1-c3ce8d1ee5e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857092646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857112645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857141083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857161212Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857188641Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.85720659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857223549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857240099Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857257098Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857274357Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_connection connected from Ok(127.0.0.1:54713)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857384921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_connection connected from Ok(127.0.0.1:47260)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857427059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_connection connected from Ok(127.0.0.1:43174)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.857458578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858119145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858152844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858169193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858183292Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858309866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858331035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858777913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858894787Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.858934045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859314566Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859345325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859360944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859476669Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859497648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859512767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859569944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859591973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[0]R flush_numbers[0..12]: [86, 97, 98, 89, 91, 95, 97, 97, 95, 94, 94, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859607222Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[0]R generation[0..12]: [49, 54, 55, 50, 52, 53, 54, 54, 53, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859621191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859635371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[1]R flush_numbers[0..12]: [86, 97, 98, 89, 91, 95, 97, 97, 95, 94, 94, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.8596493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[1]R generation[0..12]: [49, 54, 55, 50, 52, 53, 54, 54, 53, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859662949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859677019Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[2]R flush_numbers[0..12]: [86, 97, 98, 89, 91, 95, 97, 97, 95, 94, 94, 97]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859690818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[2]R generation[0..12]: [49, 54, 55, 50, 52, 53, 54, 54, 53, 52, 52, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859704337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859718247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859731426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859744495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859757185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859769974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"":"downstairs","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:06:26.859783143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859803253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"0","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859817472Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"1","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859830571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"client":"2","":"downstairs","session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859843751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.85985599Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"38876c8b-200b-4759-aaa1-c3ce8d1ee5e8 is now active with session: 7f10e446-4135-458a-a83b-69f7c30eb562","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859868869Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859880849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406,"session_id":"7f10e446-4135-458a-a83b-69f7c30eb562"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:26.859892908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 36864 data: 4 4 5 5 3 2 2 2 2 02/30 Write block 445 len 8192 data: 4 6 03/30 Write block 576 len 32768 data: 7 7 6 6 6 6 5 6 04/30 Write block 498 len 20480 data: 5 4 4 3 2 05/30 Write block 226 len 8192 data: 7 5 06/30 Write block 412 len 32768 data: 5 4 6 6 6 4 4 3 07/30 Read block 252 len 40960 08/30 Write block 54 len 4096 data: 5 09/30 Write block 553 len 40960 data: 3 2 2 3 4 7 7 7 7 6 10/30 Read block 40 len 28672 11/30 Write block 563 len 40960 data: 6 5 5 5 5 5 6 6 6 7 12/30 Read block 166 len 32768 13/30 Flush 14/30 Write block 345 len 40960 data: 5 5 5 6 8 6 6 7 8 7 15/30 Write block 306 len 12288 data: 5 7 7 16/30 Flush 17/30 Write block 414 len 24576 data: 7 7 7 5 5 4 18/30 Write block 44 len 4096 data: 9 19/30 Read block 430 len 12288 20/30 Write block 374 len 40960 data: 5 6 6 6 5 4 3 3 3 4 21/30 Read block 575 len 20480 22/30 Read block 10 len 12288 23/30 Write block 295 len 36864 data: 6 6 7 6 5 6 6 5 5 24/30 Read block 378 len 8192 25/30 Write block 166 len 32768 data: 5 5 6 6 7 7 9 7 26/30 Read block 129 len 24576 27/30 Read block 86 len 36864 28/30 Write block 156 len 28672 data: 5 5 6 6 4 5 5 29/30 Write block 525 len 24576 data: 6 6 6 5 4 4 30/30 Write block 285 len 24576 data: 5 5 8 8 7 7 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:26.879031837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 56 56 56 99 99 99 T T T 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 54 54 54 97 97 97 T T T 8 160-179 53 53 53 95 95 95 T T T 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 46 46 46 79 79 79 T T T 15 300-319 56 56 56 100 100 100 T T T 16 320-339 46 46 46 80 80 80 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 52 52 52 91 91 91 T T T 19 380-399 56 56 56 99 99 99 T T T 20 400-419 56 56 56 99 99 99 T T T 21 420-439 55 55 55 98 98 98 F F F 22 440-459 56 56 56 99 99 99 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 56 56 56 99 99 99 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 52 52 52 92 92 92 T T T 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 56, Max flush: 100 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 57 57 57 101 101 101 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 57 57 57 101 101 101 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 52 52 52 94 94 94 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 57 57 57 101 101 101 F F F 15 300-319 57 57 57 101 101 101 F F F 16 320-339 46 46 46 80 80 80 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 57 57 57 101 101 101 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 57 57 57 101 101 101 F F F 21 420-439 55 55 55 98 98 98 F F F 22 440-459 56 56 56 99 99 99 F F F 23 460-479 50 50 50 89 89 89 F F F 24 480-499 56 56 56 99 99 99 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 57 57 57 101 101 101 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 57, Max flush: 101 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.173454369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:31.216592008Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.216975419Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.249228773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.249817204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.249847613Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.249861102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.249875262Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} {"msg":"Crucible edca8e61-83c6-4947-b46c-e4b687940102 has session id: 8dca5e0a-19a6-4531-8100-48bc1c4b3b67","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.250327139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Upstairs opts: Upstairs UUID: edca8e61-83c6-4947-b46c-e4b687940102, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.250373847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Crucible stats registered with UUID: edca8e61-83c6-4947-b46c-e4b687940102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.250399036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281241859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281267708Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} {"msg":"edca8e61-83c6-4947-b46c-e4b687940102 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281330155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281369483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281418511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281458439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281493927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281531755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281568123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281603972Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281732615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281777553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:33837)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281810961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:33407)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.28183896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_connection connected from Ok(127.0.0.1:56214)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.281872908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.282403852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.282437031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.28245341Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.282476838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.282504647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.282533716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283106258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283136456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283152345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283700549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283739896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283764465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283786254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283809823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283834052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283855881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.2838825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[0]R flush_numbers[0..12]: [86, 97, 101, 89, 91, 95, 97, 101, 101, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283907328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[0]R generation[0..12]: [49, 54, 57, 50, 52, 53, 54, 57, 57, 52, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283933537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.283960866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[1]R flush_numbers[0..12]: [86, 97, 101, 89, 91, 95, 97, 101, 101, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284000554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} Wait for a query_work_queue command to finish before sending IO {"msg":"[1]R generation[0..12]: [49, 54, 57, 50, 52, 53, 54, 57, 57, 52, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284039232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.28407017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[2]R flush_numbers[0..12]: [86, 97, 101, 89, 91, 95, 97, 101, 101, 94, 94, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284098699Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[2]R generation[0..12]: [49, 54, 57, 50, 52, 53, 54, 57, 57, 52, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284127907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284157596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284184995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284215343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284243782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.28427188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"":"downstairs","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284292219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284304899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"0","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284318648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"1","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284332017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"client":"2","":"downstairs","session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284345177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284357496Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"edca8e61-83c6-4947-b46c-e4b687940102 is now active with session: 8dca5e0a-19a6-4531-8100-48bc1c4b3b67","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284370525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284382605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414,"session_id":"8dca5e0a-19a6-4531-8100-48bc1c4b3b67"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:31.284394724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 20480 data: 3 5 7 7 8 02/30 Write block 47 len 40960 data: 6 6 4 4 3 4 4 6 4 4 03/30 Read block 573 len 40960 04/30 Read block 91 len 36864 05/30 Write block 212 len 20480 data: 3 4 5 6 8 06/30 Write block 332 len 12288 data: 6 7 6 07/30 Read block 499 len 4096 08/30 Write block 472 len 28672 data: 3 3 3 2 3 3 3 09/30 Read block 65 len 36864 10/30 Read block 435 len 4096 11/30 Read block 103 len 12288 12/30 Write block 314 len 12288 data: 8 8 8 13/30 Read block 385 len 28672 14/30 Flush 15/30 Write block 441 len 36864 data: 6 7 5 5 5 7 4 4 3 16/30 Flush 17/30 Write block 364 len 40960 data: 3 4 5 5 6 8 9 7 6 6 18/30 Write block 481 len 40960 data: 6 5 3 3 2 2 2 3 3 4 19/30 Read block 244 len 12288 20/30 Read block 413 len 4096 21/30 Write block 509 len 20480 data: 6 5 5 5 5 22/30 Read block 496 len 24576 23/30 Flush 24/30 Write block 489 len 20480 data: 4 5 6 6 7 25/30 Write block 416 len 24576 data: 8 6 6 5 3 4 26/30 Read block 160 len 36864 27/30 Read block 588 len 32768 28/30 Write block 196 len 40960 data: 7 6 5 5 5 5 5 6 6 5 29/30 Read block 17 len 16384 30/30 Read block 56 len 40960 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:31.304796601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 57 57 57 101 101 101 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 52 52 52 94 94 94 T T T 10 200-219 58 58 58 102 102 102 T T T 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 58 58 58 102 102 102 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 57 57 57 101 101 101 T T T 21 420-439 55 55 55 98 98 98 T T T 22 440-459 58 58 58 103 103 103 F F F 23 460-479 58 58 58 102 102 102 F F F 24 480-499 58 58 58 104 104 104 T T T 25 500-519 58 58 58 104 104 104 F F F 26 520-539 57 57 57 101 101 101 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 58, Max flush: 104 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 86 86 86 F F F 1 020-039 54 54 54 97 97 97 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 57 57 57 101 101 101 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 59 59 59 105 105 105 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 43 43 43 75 75 75 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 58 58 58 102 102 102 F F F 16 320-339 58 58 58 102 102 102 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 59 59 59 105 105 105 F F F 21 420-439 59 59 59 105 105 105 F F F 22 440-459 58 58 58 103 103 103 F F F 23 460-479 58 58 58 102 102 102 F F F 24 480-499 59 59 59 105 105 105 F F F 25 500-519 58 58 58 104 104 104 F F F 26 520-539 57 57 57 101 101 101 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 59, Max flush: 105 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.605057727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:35.648743339Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.649142709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.681602193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682174395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682212513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682226362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682241041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} {"msg":"Crucible d8ef83ec-75c8-499d-94de-e8de78178e6f has session id: 599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682757646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Upstairs opts: Upstairs UUID: d8ef83ec-75c8-499d-94de-e8de78178e6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682793664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Crucible stats registered with UUID: d8ef83ec-75c8-499d-94de-e8de78178e6f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.682810243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713196339Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713225588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} {"msg":"d8ef83ec-75c8-499d-94de-e8de78178e6f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713290305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713314204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713347722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71337858Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713414069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713444117Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713473736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713508014Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713541592Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713561191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36779)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71358717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43387)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713800829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56706)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.713855187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714387361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714420479Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714594581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714647178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714694666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.714723434Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715088086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715117725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71542278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715778692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715808321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71582891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715852349Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715870638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.715884737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.7160361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716065868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[0]R flush_numbers[0..12]: [86, 97, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716081728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[0]R generation[0..12]: [49, 54, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716096207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716114106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[1]R flush_numbers[0..12]: [86, 97, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716128655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[1]R generation[0..12]: [49, 54, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716142765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716156924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[2]R flush_numbers[0..12]: [86, 97, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716170533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[2]R generation[0..12]: [49, 54, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716183992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716197702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716210601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71622361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.71623668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716249629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"":"downstairs","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716271178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716284678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"0","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716297887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"1","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716313096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"client":"2","":"downstairs","session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716326245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716338615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"d8ef83ec-75c8-499d-94de-e8de78178e6f is now active with session: 599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716351394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716363603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422,"session_id":"599eba4a-2c1f-4e6e-b89f-f83cbe38d4e7"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:35.716376093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 434 len 8192 data: 4 4 02/30 Read block 326 len 32768 03/30 Write block 311 len 28672 data: 7 7 9 9 9 9 6 04/30 Read block 367 len 4096 05/30 Read block 278 len 40960 06/30 Write block 421 len 24576 data: 5 5 5 5 5 6 07/30 Read block 97 len 12288 08/30 Write block 267 len 40960 data: 6 5 6 6 5 6 6 7 5 5 09/30 Read block 326 len 36864 10/30 Write block 533 len 8192 data: 6 5 11/30 Read block 304 len 4096 12/30 Write block 432 len 12288 data: 5 5 5 13/30 Write block 497 len 36864 data: 6 6 5 5 4 3 2 3 4 14/30 Read block 537 len 8192 15/30 Write block 511 len 24576 data: 6 6 6 5 4 4 16/30 Write block 24 len 28672 data: 5 5 5 3 3 3 4 17/30 Read block 8 len 20480 18/30 Write block 474 len 12288 data: 4 3 4 19/30 Write block 19 len 24576 data: 3 3 6 6 6 6 20/30 Flush 21/30 Flush 22/30 Write block 515 len 32768 data: 5 5 4 6 7 8 5 5 23/30 Write block 460 len 32768 data: 6 6 5 6 5 4 5 5 24/30 Write block 323 len 16384 data: 6 8 7 6 25/30 Write block 266 len 20480 data: 6 7 6 7 7 26/30 Write block 444 len 12288 data: 6 6 8 27/30 Read block 526 len 12288 28/30 Write block 321 len 20480 data: 5 7 7 9 8 29/30 Write block 525 len 8192 data: 7 7 30/30 Read block 414 len 16384 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 7 9 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:35.734998277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 106 106 106 F F F 1 020-039 60 60 60 106 106 106 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 57 57 57 101 101 101 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 59 59 59 105 105 105 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 60 60 60 106 106 106 T T T 14 280-299 58 58 58 102 102 102 F F F 15 300-319 60 60 60 106 106 106 F F F 16 320-339 58 58 58 102 102 102 T T T 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 59 59 59 105 105 105 F F F 21 420-439 60 60 60 106 106 106 F F F 22 440-459 58 58 58 103 103 103 T T T 23 460-479 60 60 60 106 106 106 T T T 24 480-499 60 60 60 106 106 106 F F F 25 500-519 60 60 60 106 106 106 T T T 26 520-539 60 60 60 106 106 106 T T T 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 60, Max flush: 106 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 106 106 106 F F F 1 020-039 60 60 60 106 106 106 F F F 2 040-059 58 58 58 102 102 102 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 F F F 6 120-139 54 54 54 97 97 97 F F F 7 140-159 57 57 57 101 101 101 F F F 8 160-179 57 57 57 101 101 101 F F F 9 180-199 59 59 59 105 105 105 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 60 60 60 106 106 106 F F F 16 320-339 61 61 61 107 107 107 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 59 59 59 105 105 105 F F F 21 420-439 60 60 60 106 106 106 F F F 22 440-459 61 61 61 107 107 107 F F F 23 460-479 61 61 61 107 107 107 F F F 24 480-499 60 60 60 106 106 106 F F F 25 500-519 61 61 61 107 107 107 F F F 26 520-539 61 61 61 107 107 107 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 F F F Max gen: 61, Max flush: 107 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.045833471Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.090906294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.09160887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.091646748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.091661407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.091676736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} {"msg":"Crucible 007c5a69-b669-4f89-903a-03e43583a2ba has session id: 4bd38175-0e90-483a-b132-28d840582f86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.092064617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Upstairs opts: Upstairs UUID: 007c5a69-b669-4f89-903a-03e43583a2ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.092111015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Crucible stats registered with UUID: 007c5a69-b669-4f89-903a-03e43583a2ba","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.092136514Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123241684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123272753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} {"msg":"007c5a69-b669-4f89-903a-03e43583a2ba active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.12332131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.12334066Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123379187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123398287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123419406Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123436705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123453924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123470453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123487842Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123504851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:46096)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123613176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:52003)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123662774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_connection connected from Ok(127.0.0.1:41356)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.123865964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124400917Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124430456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124445405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124459645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124473544Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.124487283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125124822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.12516812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125203978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125669435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125694964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125709823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125784079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125804478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125818558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125831037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125845126Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[0]R flush_numbers[0..12]: [106, 106, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125859286Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[0]R generation[0..12]: [60, 60, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125873215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125887294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[1]R flush_numbers[0..12]: [106, 106, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125900954Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[1]R generation[0..12]: [60, 60, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125918473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125933232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[2]R flush_numbers[0..12]: [106, 106, 102, 89, 91, 95, 97, 101, 101, 105, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125947021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[2]R generation[0..12]: [60, 60, 58, 50, 52, 53, 54, 57, 57, 59, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.12596937Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.125997449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126024717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126049146Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"Wait for a query_work_queue command to finish before sending IO } {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126082685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126110713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"":"downstairs","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126131692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126144272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"0","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126160711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"1","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.12618535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"client":"2","":"downstairs","session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126213108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126235317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"007c5a69-b669-4f89-903a-03e43583a2ba is now active with session: 4bd38175-0e90-483a-b132-28d840582f86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126255906Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126279635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430,"session_id":"4bd38175-0e90-483a-b132-28d840582f86"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:40.126304024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 134 len 32768 data: 7 6 6 6 7 6 5 6 02/30 Read block 153 len 40960 03/30 Read block 53 len 40960 04/30 Write block 479 len 40960 data: 3 4 7 6 4 4 3 3 3 4 05/30 Read block 534 len 12288 06/30 Read block 230 len 24576 07/30 Flush 08/30 Write block 10 len 8192 data: 5 4 09/30 Read block 330 len 16384 10/30 Write block 16 len 36864 data: 4 3 3 4 4 7 7 7 7 11/30 Write block 182 len 20480 data: 5 4 5 5 3 12/30 Write block 414 len 36864 data: 8 8 9 7 7 6 4 6 6 13/30 Read block 286 len 4096 14/30 Read block 517 len 4096 15/30 Read block 436 len 12288 16/30 Read block 372 len 24576 17/30 Write block 104 len 16384 data: 7 10 9 9 18/30 Write block 136 len 32768 data: 7 7 8 7 6 7 6 7 19/30 Write block 35 len 32768 data: 8 8 8 6 5 6 6 7 20/30 Write block 185 len 24576 data: 6 4 2 2 3 3 21/30 Write block 435 len 4096 data: 5 22/30 Write block 595 len 16384 data: 2 2 2 2 23/30 Read block 127 len 36864 24/30 Write block 453 len 32768 data: 2 2 4 4 4 4 6 7 25/30 Read block 447 len 32768 26/30 Write block 589 len 28672 data: 7 6 6 5 4 3 3 27/30 Write block 178 len 32768 data: 5 6 6 5 6 5 6 7 28/30 Read block 440 len 20480 29/30 Read block 93 len 8192 30/30 Write block 56 len 12288 data: 5 4 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:40.145996546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 106 106 106 T T T 1 020-039 60 60 60 106 106 106 T T T 2 040-059 58 58 58 102 102 102 T T T 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 53 53 53 95 95 95 T T T 6 120-139 62 62 62 108 108 108 T T T 7 140-159 62 62 62 108 108 108 T T T 8 160-179 57 57 57 101 101 101 T T T 9 180-199 59 59 59 105 105 105 T T T 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 60 60 60 106 106 106 F F F 16 320-339 61 61 61 107 107 107 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 59 59 59 105 105 105 T T T 21 420-439 60 60 60 106 106 106 T T T 22 440-459 61 61 61 107 107 107 T T T 23 460-479 62 62 62 108 108 108 T T T 24 480-499 62 62 62 108 108 108 F F F 25 500-519 61 61 61 107 107 107 F F F 26 520-539 61 61 61 107 107 107 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 56 56 56 99 99 99 T T T Max gen: 62, Max flush: 108 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 109 109 109 F F F 1 020-039 63 63 63 109 109 109 F F F 2 040-059 63 63 63 109 109 109 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 63 63 63 109 109 109 F F F 6 120-139 63 63 63 109 109 109 F F F 7 140-159 63 63 63 109 109 109 F F F 8 160-179 63 63 63 109 109 109 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 52 52 52 94 94 94 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 60 60 60 106 106 106 F F F 16 320-339 61 61 61 107 107 107 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 62 62 62 108 108 108 F F F 25 500-519 61 61 61 107 107 107 F F F 26 520-539 61 61 61 107 107 107 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 63, Max flush: 109 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.461325369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.506392863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.507005243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.50705403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.50706993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.507084809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} {"msg":"Crucible a006764a-d710-4b56-a3c2-2aabd87415c4 has session id: 6674c5db-e5b1-4559-a10b-a59940fb4cc4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.507547716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Upstairs opts: Upstairs UUID: a006764a-d710-4b56-a3c2-2aabd87415c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.507589234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Crucible stats registered with UUID: a006764a-d710-4b56-a3c2-2aabd87415c4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.507606653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.53797494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538008528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} {"msg":"a006764a-d710-4b56-a3c2-2aabd87415c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538084314Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538107513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538135892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538158511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.53817598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538196549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538227258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538260126Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538294274Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538331833Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:60668)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538365621Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63731)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538501574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52828)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.538546062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539049107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539078676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539226449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539264207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539294055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539318974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539693635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539776951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.539995161Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.54021976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540249258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540264867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540333324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540353043Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540366922Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540506706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540525585Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[0]R flush_numbers[0..12]: [109, 109, 109, 89, 91, 109, 109, 109, 109, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540545284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[0]R generation[0..12]: [63, 63, 63, 50, 52, 63, 63, 63, 63, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540559303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540573382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[1]R flush_numbers[0..12]: [109, 109, 109, 89, 91, 109, 109, 109, 109, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540587371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[1]R generation[0..12]: [63, 63, 63, 50, 52, 63, 63, 63, 63, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540603571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.54061789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[2]R flush_numbers[0..12]: [109, 109, 109, 89, 91, 109, 109, 109, 109, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540631629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[2]R generation[0..12]: [63, 63, 63, 50, 52, 63, 63, 63, 63, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540645259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540659038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540672087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540685287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540698226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540711225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"":"downstairs","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:06:44.540724385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540742154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"0","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540755273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"1","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540768373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"client":"2","":"downstairs","session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540781552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540795511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"a006764a-d710-4b56-a3c2-2aabd87415c4 is now active with session: 6674c5db-e5b1-4559-a10b-a59940fb4cc4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540808391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.54082047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438,"session_id":"6674c5db-e5b1-4559-a10b-a59940fb4cc4"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:44.540832839Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 40960 data: 5 4 5 5 7 5 6 5 6 5 02/30 Read block 160 len 28672 03/30 Write block 142 len 40960 data: 7 8 6 5 4 4 2 2 2 3 04/30 Flush 05/30 Write block 250 len 16384 data: 5 4 5 5 06/30 Read block 366 len 36864 07/30 Write block 319 len 12288 data: 6 5 6 08/30 Write block 39 len 24576 data: 6 7 7 8 9 10 09/30 Read block 455 len 36864 10/30 Read block 462 len 8192 11/30 Flush 12/30 Write block 105 len 4096 data: 11 13/30 Read block 562 len 4096 14/30 Write block 129 len 16384 data: 7 8 7 5 15/30 Write block 13 len 36864 data: 4 5 4 5 4 4 5 5 8 16/30 Read block 253 len 16384 17/30 Read block 289 len 8192 18/30 Flush 19/30 Write block 518 len 40960 data: 7 8 9 6 6 5 6 8 8 7 20/30 Read block 449 len 16384 21/30 Write block 173 len 16384 data: 8 8 8 6 22/30 Read block 6 len 40960 23/30 Write block 249 len 32768 data: 5 6 5 6 6 3 4 3 24/30 Read block 163 len 8192 25/30 Write block 391 len 16384 data: 5 6 6 4 26/30 Read block 140 len 28672 27/30 Read block 590 len 8192 28/30 Read block 383 len 20480 29/30 Read block 314 len 16384 30/30 Read block 168 len 20480 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:44.561091973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 112 112 112 F F F 1 020-039 64 64 64 112 112 112 F F F 2 040-059 64 64 64 111 111 111 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 64 64 64 112 112 112 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 63 63 63 109 109 109 T T T 9 180-199 63 63 63 109 109 109 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 64 64 64 111 111 111 T T T 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 57 57 57 101 101 101 T T T 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 62 62 62 108 108 108 F F F 25 500-519 61 61 61 107 107 107 T T T 26 520-539 61 61 61 107 107 107 T T T 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 64, Max flush: 112 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 112 112 112 F F F 1 020-039 64 64 64 112 112 112 F F F 2 040-059 64 64 64 111 111 111 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 64 64 64 112 112 112 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 59 59 59 105 105 105 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 65 65 65 113 113 113 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 56 56 56 100 100 100 F F F 18 360-379 58 58 58 104 104 104 F F F 19 380-399 65 65 65 113 113 113 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 62 62 62 108 108 108 F F F 25 500-519 65 65 65 113 113 113 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 56 56 56 99 99 99 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 65, Max flush: 113 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.831836928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.877820027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.878506564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.878549992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.878564971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.8785798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} {"msg":"Crucible 6a507b4b-f25d-433a-837e-495a0a806f58 has session id: 29bf2fc3-de85-421d-8c38-16616d0700d5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.879079836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Upstairs opts: Upstairs UUID: 6a507b4b-f25d-433a-837e-495a0a806f58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.879115464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Crucible stats registered with UUID: 6a507b4b-f25d-433a-837e-495a0a806f58","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.879131893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.9103161Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910367477Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} {"msg":"6a507b4b-f25d-433a-837e-495a0a806f58 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910390836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910410985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910457363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910493771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910528329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910561178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910599906Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910633044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910664233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910698371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37089)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910732069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62260)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910778777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41925)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.910807896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911543189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911573808Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911588907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911603117Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911616926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.911630535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912154799Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912242415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912464594Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912707162Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.91274858Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.91276352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912826236Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912845425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.912859225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913002768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913021347Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[0]R flush_numbers[0..12]: [112, 112, 111, 89, 91, 112, 112, 110, 113, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913041436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[0]R generation[0..12]: [64, 64, 64, 50, 52, 64, 64, 64, 65, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913055595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913070804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[1]R flush_numbers[0..12]: [112, 112, 111, 89, 91, 112, 112, 110, 113, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913084944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[1]R generation[0..12]: [64, 64, 64, 50, 52, 64, 64, 64, 65, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913098433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913112382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[2]R flush_numbers[0..12]: [112, 112, 111, 89, 91, 112, 112, 110, 113, 109, 105, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913129062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[2]R generation[0..12]: [64, 64, 64, 50, 52, 64, 64, 64, 65, 63, 59, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913142831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.91315661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.9131697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913183199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913196238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913209407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"":"downstairs","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913222647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913243126Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"0","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913256645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"1","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913269755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"client":"2","":"downstairs","session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913283074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913295433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"6a507b4b-f25d-433a-837e-495a0a806f58 is now active with session: 29bf2fc3-de85-421d-8c38-16616d0700d5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913308413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913320402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446,"session_id":"29bf2fc3-de85-421d-8c38-16616d0700d5"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:48.913334881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 497 len 4096 data: 7 02/30 Read block 16 len 4096 03/30 Read block 241 len 40960 04/30 Write block 27 len 24576 data: 4 4 4 5 3 4 05/30 Write block 124 len 4096 data: 4 06/30 Read block 8 len 24576 07/30 Flush 08/30 Write block 547 len 4096 data: 6 09/30 Read block 220 len 36864 10/30 Read block 516 len 40960 11/30 Read block 54 len 36864 12/30 Read block 215 len 24576 13/30 Read block 481 len 4096 14/30 Read block 414 len 16384 15/30 Read block 158 len 4096 16/30 Read block 590 len 4096 17/30 Read block 492 len 24576 18/30 Write block 359 len 32768 data: 4 4 5 6 4 4 5 6 19/30 Read block 254 len 40960 20/30 Read block 354 len 36864 21/30 Read block 211 len 28672 22/30 Write block 36 len 36864 data: 9 9 7 7 8 8 9 10 11 23/30 Write block 206 len 32768 data: 4 4 2 2 3 4 4 5 24/30 Read block 352 len 20480 25/30 Read block 386 len 24576 26/30 Flush 27/30 Write block 515 len 20480 data: 6 6 5 8 9 28/30 Read block 561 len 12288 29/30 Write block 126 len 32768 data: 5 6 6 8 9 8 6 6 30/30 Read block 75 len 4096 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:48.932936948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 112 112 112 F F F 1 020-039 66 66 66 115 115 115 F F F 2 040-059 66 66 66 115 115 115 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 66 66 66 114 114 114 T T T 7 140-159 64 64 64 110 110 110 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 65 65 65 113 113 113 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 66 66 66 115 115 115 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 65 65 65 113 113 113 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 65 65 65 113 113 113 T T T 26 520-539 65 65 65 113 113 113 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 66, Max flush: 115 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 112 112 112 F F F 1 020-039 66 66 66 115 115 115 F F F 2 040-059 66 66 66 115 115 115 F F F 3 060-079 50 50 50 89 89 89 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 65 65 65 113 113 113 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 56 56 56 99 99 99 F F F 12 240-259 65 65 65 113 113 113 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 66 66 66 115 115 115 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 65 65 65 113 113 113 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 67 67 67 116 116 116 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 56 56 56 99 99 99 F F F 29 580-599 63 63 63 109 109 109 F F F Max gen: 67, Max flush: 116 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.185779543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:53.229709983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.230111353Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.262581347Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263176777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263207676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263224625Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263239414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} {"msg":"Crucible 07ee6418-b6ac-4dc3-9b3c-4d484d125745 has session id: 53310310-a8e9-479f-9ae8-15b0345ee467","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263788918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Upstairs opts: Upstairs UUID: 07ee6418-b6ac-4dc3-9b3c-4d484d125745, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263823046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Crucible stats registered with UUID: 07ee6418-b6ac-4dc3-9b3c-4d484d125745","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.263838415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.294986044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295031021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} {"msg":"07ee6418-b6ac-4dc3-9b3c-4d484d125745 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295099028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295127457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295151665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295166545Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295180504Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295193933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295207543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295220602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295234371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295251571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_connection connected from Ok(127.0.0.1:40184)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295367935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_connection connected from Ok(127.0.0.1:35890)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295419742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_connection connected from Ok(127.0.0.1:59016)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.295490059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296132907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296161316Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296176415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296190274Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296204194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296218413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296805264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296849122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.296922108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297381986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297410564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297428344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.29749512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297516629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297530188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297542348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297555697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[0]R flush_numbers[0..12]: [112, 115, 115, 89, 91, 112, 116, 110, 113, 109, 115, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297569526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[0]R generation[0..12]: [64, 66, 66, 50, 52, 64, 67, 64, 65, 63, 66, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297583426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297597495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[1]R flush_numbers[0..12]: [112, 115, 115, 89, 91, 112, 116, 110, 113, 109, 115, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297611164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[1]R generation[0..12]: [64, 66, 66, 50, 52, 64, 67, 64, 65, 63, 66, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297624734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297638603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[2]R flush_numbers[0..12]: [112, 115, 115, 89, 91, 112, 116, 110, 113, 109, 115, 99]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297652353Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[2]R generation[0..12]: [64, 66, 66, 50, 52, 64, 67, 64, 65, 63, 66, 56]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297665852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297683791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.29769717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297710459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297723399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-11-26T22:06:53.297736318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"":"downstairs","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297757897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297770047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"0","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297783526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"1","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297796905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"client":"2","":"downstairs","session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297810365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297822614Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"07ee6418-b6ac-4dc3-9b3c-4d484d125745 is now active with session: 53310310-a8e9-479f-9ae8-15b0345ee467","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297835404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297847423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454,"session_id":"53310310-a8e9-479f-9ae8-15b0345ee467"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:53.297859672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 20480 data: 10 9 8 8 7 02/30 Write block 154 len 36864 data: 4 5 6 6 7 7 5 6 6 03/30 Write block 20 len 20480 data: 6 9 8 8 8 04/30 Read block 228 len 36864 05/30 Read block 500 len 20480 06/30 Read block 93 len 20480 07/30 Write block 49 len 16384 data: 5 6 5 6 08/30 Write block 594 len 16384 data: 4 4 3 3 09/30 Write block 155 len 28672 data: 6 7 7 8 8 6 7 10/30 Read block 369 len 24576 11/30 Read block 267 len 24576 12/30 Read block 135 len 40960 13/30 Read block 132 len 32768 14/30 Read block 468 len 40960 15/30 Read block 93 len 36864 16/30 Write block 69 len 36864 data: 7 7 5 5 6 7 6 6 5 17/30 Write block 166 len 20480 data: 6 6 7 7 8 18/30 Flush 19/30 Flush 20/30 Write block 344 len 8192 data: 3 6 21/30 Write block 283 len 4096 data: 4 22/30 Read block 463 len 32768 23/30 Write block 234 len 20480 data: 7 9 9 9 6 24/30 Read block 358 len 32768 25/30 Read block 561 len 4096 26/30 Read block 88 len 24576 27/30 Read block 74 len 8192 28/30 Read block 23 len 28672 29/30 Write block 14 len 12288 data: 6 5 6 30/30 Write block 7 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:53.316926945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 112 112 112 T T T 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 68 68 68 117 117 117 F F F 8 160-179 68 68 68 117 117 117 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 56 56 56 99 99 99 T T T 12 240-259 65 65 65 113 113 113 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 58 58 58 102 102 102 T T T 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 66 66 66 115 115 115 T T T 18 360-379 66 66 66 115 115 115 F F F 19 380-399 65 65 65 113 113 113 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 67 67 67 116 116 116 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 68 68 68 117 117 117 F F F Max gen: 68, Max flush: 117 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 68 68 68 117 117 117 F F F 4 080-099 52 52 52 91 91 91 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 68 68 68 117 117 117 F F F 8 160-179 68 68 68 117 117 117 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 65 65 65 113 113 113 F F F 13 260-279 61 61 61 107 107 107 F F F 14 280-299 69 69 69 118 118 118 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 69 69 69 118 118 118 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 65 65 65 113 113 113 F F F 20 400-419 63 63 63 109 109 109 F F F 21 420-439 63 63 63 109 109 109 F F F 22 440-459 63 63 63 109 109 109 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 67 67 67 116 116 116 F F F 26 520-539 65 65 65 113 113 113 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 68 68 68 117 117 117 F F F Max gen: 69, Max flush: 118 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.610496389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:57.655201181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.655587142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.688677685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689266766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689302004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689316153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689330653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} {"msg":"Crucible a1b8f746-2f3e-426b-b42c-82fbb636608f has session id: a021fe8f-67b2-4271-8549-67268793014d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689915044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Upstairs opts: Upstairs UUID: a1b8f746-2f3e-426b-b42c-82fbb636608f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.689962791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Crucible stats registered with UUID: a1b8f746-2f3e-426b-b42c-82fbb636608f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.6899887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.720956458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.720986836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} {"msg":"a1b8f746-2f3e-426b-b42c-82fbb636608f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721056953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721079172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72111267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721145118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721179577Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721215035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721249963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721284851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72132081Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721477042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41509)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72151841Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57181)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721548378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37581)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.721577457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722203036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722234675Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722261783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722377818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722422995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722457894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722856054Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.722898502Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723041755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723435236Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723460735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723476744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723568079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723591438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723605777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723618137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723631736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 117, 91, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723656885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[0]R generation[0..12]: [69, 68, 68, 68, 52, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723671464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723685643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 117, 91, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723700203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[1]R generation[0..12]: [69, 68, 68, 68, 52, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723713952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723727971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 117, 91, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723741811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[2]R generation[0..12]: [69, 68, 68, 68, 52, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72375536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723769549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723785908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723799308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723812317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723825366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"":"downstairs","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723898223Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723916182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"0","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723929971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"1","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723943131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"client":"2","":"downstairs","session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72395651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.72396885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"a1b8f746-2f3e-426b-b42c-82fbb636608f is now active with session: a021fe8f-67b2-4271-8549-67268793014d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723981759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.723993848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462,"session_id":"a021fe8f-67b2-4271-8549-67268793014d"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:06:57.724006278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 581 len 28672 data: 7 6 7 5 7 6 6 02/30 Read block 218 len 24576 03/30 Read block 439 len 8192 04/30 Flush 05/30 Write block 523 len 40960 data: 6 7 9 9 8 6 5 5 6 7 06/30 Read block 515 len 40960 07/30 Write block 529 len 16384 data: 6 6 7 8 08/30 Write block 92 len 20480 data: 7 5 4 5 6 09/30 Write block 251 len 12288 data: 6 7 7 10/30 Write block 514 len 32768 data: 6 7 7 6 9 10 10 7 11/30 Read block 326 len 8192 12/30 Write block 535 len 4096 data: 7 13/30 Read block 384 len 32768 14/30 Read block 346 len 40960 15/30 Write block 457 len 4096 data: 5 16/30 Write block 79 len 4096 data: 2 17/30 Write block 429 len 4096 data: 3 18/30 Read block 126 len 4096 19/30 Write block 351 len 24576 data: 7 8 9 8 6 4 20/30 Write block 437 len 24576 data: 5 6 7 8 7 8 21/30 Read block 144 len 12288 22/30 Read block 89 len 4096 23/30 Write block 297 len 12288 data: 9 7 6 24/30 Flush 25/30 Write block 393 len 8192 data: 7 5 26/30 Read block 27 len 4096 27/30 Read block 161 len 16384 28/30 Write block 407 len 16384 data: 7 7 9 5 29/30 Write block 258 len 28672 data: 3 2 6 7 8 7 7 30/30 Read block 122 len 20480 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:06:57.74206706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 120 120 120 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 68 68 68 117 117 117 F F F 8 160-179 68 68 68 117 117 117 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 120 120 120 T T T 13 260-279 61 61 61 107 107 107 T T T 14 280-299 70 70 70 120 120 120 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 70 70 70 120 120 120 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 65 65 65 113 113 113 T T T 20 400-419 63 63 63 109 109 109 T T T 21 420-439 70 70 70 120 120 120 F F F 22 440-459 70 70 70 120 120 120 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 70 70 70 119 119 119 F F F Max gen: 70, Max flush: 120 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 120 120 120 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 68 68 68 117 117 117 F F F 8 160-179 68 68 68 117 117 117 F F F 9 180-199 63 63 63 109 109 109 F F F 10 200-219 66 66 66 115 115 115 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 71 71 71 121 121 121 F F F 13 260-279 71 71 71 121 121 121 F F F 14 280-299 70 70 70 120 120 120 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 70 70 70 120 120 120 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 71 71 71 121 121 121 F F F 20 400-419 71 71 71 121 121 121 F F F 21 420-439 70 70 70 120 120 120 F F F 22 440-459 70 70 70 120 120 120 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 F F F 25 500-519 70 70 70 120 120 120 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 66 66 66 115 115 115 F F F 28 560-579 68 68 68 117 117 117 F F F 29 580-599 70 70 70 119 119 119 F F F Max gen: 71, Max flush: 121 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.039018706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.08410473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.084787676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.084830744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.084846593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.084861783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} {"msg":"Crucible ed12df24-d8a7-4cce-9d45-f25c649d387a has session id: f74b699b-a439-43d7-a258-f3df90a93146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.085398406Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Upstairs opts: Upstairs UUID: ed12df24-d8a7-4cce-9d45-f25c649d387a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.085434294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Crucible stats registered with UUID: ed12df24-d8a7-4cce-9d45-f25c649d387a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.085453833Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116370703Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116404981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} {"msg":"ed12df24-d8a7-4cce-9d45-f25c649d387a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.11644191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116462809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116505567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116542405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116578743Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116614271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116650049Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116676278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116708106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116740045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_connection connected from Ok(127.0.0.1:34119)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116771383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_connection connected from Ok(127.0.0.1:44841)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.11684851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_connection connected from Ok(127.0.0.1:44696)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.116893717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117504747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117546145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117577854Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117608232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117639681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.117669039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118196713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118241071Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118276869Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118848961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.1188791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118895099Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118907998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118921848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118935147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118947616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118964926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 120, 120, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118979385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[0]R generation[0..12]: [69, 68, 68, 70, 70, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.118993424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119018023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 120, 120, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119039582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[1]R generation[0..12]: [69, 68, 68, 70, 70, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119053901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.11906807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 120, 120, 112, 116, 117, 117, 109, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.1190848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[2]R generation[0..12]: [69, 68, 68, 70, 70, 64, 67, 68, 68, 63, 66, 69]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:07:02.119110839Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119140217Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119153666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119167796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119184615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119208424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"":"downstairs","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119227273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119239892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"0","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119255061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"1","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.11927946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"client":"2","":"downstairs","session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119297909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119310539Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"ed12df24-d8a7-4cce-9d45-f25c649d387a is now active with session: f74b699b-a439-43d7-a258-f3df90a93146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119323578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119341137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470,"session_id":"f74b699b-a439-43d7-a258-f3df90a93146"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:02.119364036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 12288 data: 5 5 5 02/30 Read block 535 len 12288 03/30 Read block 293 len 40960 04/30 Read block 295 len 28672 05/30 Write block 557 len 8192 data: 5 8 06/30 Write block 158 len 28672 data: 9 9 7 8 7 4 6 07/30 Write block 545 len 16384 data: 7 7 7 6 08/30 Write block 590 len 36864 data: 7 7 6 5 5 5 4 4 3 09/30 Read block 513 len 12288 10/30 Write block 348 len 4096 data: 7 11/30 Flush 12/30 Write block 565 len 40960 data: 6 6 6 6 7 7 7 8 8 11 13/30 Write block 241 len 40960 data: 4 5 4 5 5 6 6 5 6 7 14/30 Read block 521 len 32768 15/30 Flush 16/30 Write block 181 len 32768 data: 6 7 6 7 8 5 3 3 17/30 Write block 241 len 16384 data: 5 6 5 6 18/30 Read block 350 len 20480 19/30 Write block 453 len 24576 data: 3 3 5 5 6 5 20/30 Read block 476 len 4096 21/30 Read block 133 len 8192 22/30 Read block 337 len 28672 23/30 Read block 449 len 40960 24/30 Read block 441 len 24576 25/30 Write block 218 len 8192 data: 8 9 26/30 Write block 177 len 4096 data: 5 27/30 Write block 343 len 16384 data: 6 4 7 6 28/30 Write block 495 len 28672 data: 4 6 8 7 6 6 5 29/30 Write block 88 len 40960 data: 7 7 8 8 8 6 5 6 7 5 30/30 Read block 326 len 4096 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 15 13 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:02.138344033Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 120 120 120 T T T 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 72 72 72 122 122 122 T T T 9 180-199 63 63 63 109 109 109 T T T 10 200-219 66 66 66 115 115 115 T T T 11 220-239 69 69 69 118 118 118 F F F 12 240-259 72 72 72 123 123 123 T T T 13 260-279 71 71 71 121 121 121 F F F 14 280-299 70 70 70 120 120 120 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 72 72 72 122 122 122 T T T 18 360-379 66 66 66 115 115 115 F F F 19 380-399 71 71 71 121 121 121 F F F 20 400-419 71 71 71 121 121 121 F F F 21 420-439 70 70 70 120 120 120 F F F 22 440-459 70 70 70 120 120 120 T T T 23 460-479 63 63 63 109 109 109 F F F 24 480-499 66 66 66 114 114 114 T T T 25 500-519 70 70 70 120 120 120 T T T 26 520-539 70 70 70 120 120 120 F F F 27 540-559 72 72 72 122 122 122 F F F 28 560-579 72 72 72 123 123 123 F F F 29 580-599 72 72 72 122 122 122 F F F Max gen: 72, Max flush: 123 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 73 73 73 124 124 124 F F F 5 100-119 64 64 64 112 112 112 F F F 6 120-139 67 67 67 116 116 116 F F F 7 140-159 72 72 72 122 122 122 F F F 8 160-179 73 73 73 124 124 124 F F F 9 180-199 73 73 73 124 124 124 F F F 10 200-219 73 73 73 124 124 124 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 71 71 71 121 121 121 F F F 14 280-299 70 70 70 120 120 120 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 73 73 73 124 124 124 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 71 71 71 121 121 121 F F F 20 400-419 71 71 71 121 121 121 F F F 21 420-439 70 70 70 120 120 120 F F F 22 440-459 73 73 73 124 124 124 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 72 72 72 122 122 122 F F F 28 560-579 72 72 72 123 123 123 F F F 29 580-599 72 72 72 122 122 122 F F F Max gen: 73, Max flush: 124 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.434586435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.479146895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.479777664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.479815782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.479830911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.47984581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} {"msg":"Crucible f2474ca9-1c61-477e-854b-05efa0de5c47 has session id: f118b662-6839-4249-8490-a532e96e70ba","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.480274419Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Upstairs opts: Upstairs UUID: f2474ca9-1c61-477e-854b-05efa0de5c47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.480320127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Crucible stats registered with UUID: f2474ca9-1c61-477e-854b-05efa0de5c47","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.480345096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510585779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510626437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} {"msg":"f2474ca9-1c61-477e-854b-05efa0de5c47 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510655635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510669945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510692053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510706203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510719722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510733011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510746571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.51075996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510780649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510881884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:40321)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.510927652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:48134)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.51096391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:40457)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511113823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511536162Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.51157385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511607668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511639027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511807069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.511835797Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512300594Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512331973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512349182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512881856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512910154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512925194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512937933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512951372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512964562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512976671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.512989871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 120, 124, 112, 116, 122, 124, 124, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513015029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[0]R generation[0..12]: [69, 68, 68, 70, 73, 64, 67, 72, 73, 73, 73, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513035848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513050628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 120, 124, 112, 116, 122, 124, 124, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513064587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[1]R generation[0..12]: [69, 68, 68, 70, 73, 64, 67, 72, 73, 73, 73, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513078276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513092335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 120, 124, 112, 116, 122, 124, 124, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513106065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[2]R generation[0..12]: [69, 68, 68, 70, 73, 64, 67, 72, 73, 73, 73, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513119594Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513133523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513153802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513167232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513181281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.5131943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"":"downstairs","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.51320777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513222189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"0","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513236048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"1","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513249428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"client":"2","":"downstairs","session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513262887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513275217Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"f2474ca9-1c61-477e-854b-05efa0de5c47 is now active with session: f118b662-6839-4249-8490-a532e96e70ba","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513288006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513300265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478,"session_id":"f118b662-6839-4249-8490-a532e96e70ba"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:06.513312535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 113 len 32768 data: 7 5 6 7 6 6 5 5 02/30 Write block 193 len 36864 data: 6 6 7 8 7 6 6 6 6 03/30 Write block 402 len 8192 data: 5 5 04/30 Read block 124 len 32768 05/30 Read block 67 len 32768 06/30 Read block 339 len 36864 07/30 Write block 147 len 8192 data: 5 3 08/30 Write block 584 len 28672 data: 6 8 7 7 7 8 8 09/30 Read block 364 len 40960 10/30 Write block 69 len 20480 data: 8 8 6 6 7 11/30 Write block 371 len 32768 data: 8 7 7 6 7 7 7 6 12/30 Flush 13/30 Write block 577 len 32768 data: 9 8 7 7 8 7 8 7 14/30 Read block 233 len 24576 15/30 Read block 594 len 12288 16/30 Read block 486 len 32768 17/30 Read block 385 len 8192 18/30 Read block 489 len 28672 19/30 Write block 277 len 4096 data: 6 20/30 Write block 141 len 12288 data: 8 8 9 21/30 Write block 211 len 4096 data: 5 22/30 Write block 173 len 20480 data: 9 9 9 7 6 23/30 Write block 138 len 16384 data: 9 8 7 9 24/30 Write block 137 len 36864 data: 8 10 9 8 10 9 10 7 6 25/30 Read block 151 len 4096 26/30 Write block 99 len 24576 data: 4 4 6 7 7 8 27/30 Read block 453 len 28672 28/30 Write block 288 len 40960 data: 9 8 8 5 5 4 6 8 8 10 29/30 Write block 556 len 36864 data: 4 6 9 8 8 8 7 7 6 30/30 Read block 410 len 8192 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 16 16 18 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:06.532450594Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 74 74 74 125 125 125 F F F 4 080-099 73 73 73 124 124 124 T T T 5 100-119 74 74 74 125 125 125 T T T 6 120-139 74 74 74 125 125 125 T T T 7 140-159 74 74 74 125 125 125 T T T 8 160-179 73 73 73 124 124 124 T T T 9 180-199 74 74 74 125 125 125 F F F 10 200-219 74 74 74 125 125 125 T T T 11 220-239 69 69 69 118 118 118 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 71 71 71 121 121 121 T T T 14 280-299 70 70 70 120 120 120 T T T 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 73 73 73 124 124 124 F F F 18 360-379 74 74 74 125 125 125 F F F 19 380-399 71 71 71 121 121 121 F F F 20 400-419 74 74 74 125 125 125 F F F 21 420-439 70 70 70 120 120 120 F F F 22 440-459 73 73 73 124 124 124 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 72 72 72 122 122 122 T T T 28 560-579 72 72 72 123 123 123 T T T 29 580-599 74 74 74 125 125 125 T T T Max gen: 74, Max flush: 125 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 74 74 74 125 125 125 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 75 75 75 126 126 126 F F F 6 120-139 75 75 75 126 126 126 F F F 7 140-159 75 75 75 126 126 126 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 75 75 75 126 126 126 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 75 75 75 126 126 126 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 F F F 17 340-359 73 73 73 124 124 124 F F F 18 360-379 74 74 74 125 125 125 F F F 19 380-399 71 71 71 121 121 121 F F F 20 400-419 74 74 74 125 125 125 F F F 21 420-439 70 70 70 120 120 120 F F F 22 440-459 73 73 73 124 124 124 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 F F F 28 560-579 75 75 75 126 126 126 F F F 29 580-599 75 75 75 126 126 126 F F F Max gen: 75, Max flush: 126 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.834629774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.879998953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.880690579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.880728178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.880742277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.880756386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} {"msg":"Crucible d4ab996f-9f2b-469f-906f-096a4c24ba25 has session id: 14e213b1-cf47-4335-9e89-b33fe23d6d31","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.881223983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Upstairs opts: Upstairs UUID: d4ab996f-9f2b-469f-906f-096a4c24ba25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.881269331Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Crucible stats registered with UUID: d4ab996f-9f2b-469f-906f-096a4c24ba25","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.88129164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911811399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911845348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} {"msg":"d4ab996f-9f2b-469f-906f-096a4c24ba25 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911874946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911892735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911920454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911938683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911956152Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911973091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.911990411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912031009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912124354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912156003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:55090)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912185761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:57396)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912213749Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_connection connected from Ok(127.0.0.1:55092)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.912243008Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.91301778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913059428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913096286Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913116025Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913130884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913144814Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913694867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913722045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.913737165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.91423859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914269828Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914285898Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914299147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914312966Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914326596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914338945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914352175Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[0]R flush_numbers[0..12]: [118, 117, 117, 125, 126, 126, 126, 126, 126, 125, 126, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914365984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[0]R generation[0..12]: [69, 68, 68, 74, 75, 75, 75, 75, 75, 74, 75, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914382923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914397372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[1]R flush_numbers[0..12]: [118, 117, 117, 125, 126, 126, 126, 126, 126, 125, 126, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914411321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[1]R generation[0..12]: [69, 68, 68, 74, 75, 75, 75, 75, 75, 74, 75, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914424941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.91443897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[2]R flush_numbers[0..12]: [118, 117, 117, 125, 126, 126, 126, 126, 126, 125, 126, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914452739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[2]R generation[0..12]: [69, 68, 68, 74, 75, 75, 75, 75, 75, 74, 75, 69]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914466269Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-26T22:07:10.914487768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914501057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914514147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914527296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914540335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"":"downstairs","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914553465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914565484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"0","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914580463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"1","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914593723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"client":"2","":"downstairs","session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914607052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914619391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"d4ab996f-9f2b-469f-906f-096a4c24ba25 is now active with session: 14e213b1-cf47-4335-9e89-b33fe23d6d31","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.914632351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.91464442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486,"session_id":"14e213b1-cf47-4335-9e89-b33fe23d6d31"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:10.91465662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 109 len 32768 data: 10 10 11 10 8 6 7 8 02/30 Write block 396 len 36864 data: 3 3 3 3 4 5 6 6 6 03/30 Read block 562 len 8192 04/30 Write block 381 len 40960 data: 4 4 5 6 7 7 6 4 4 5 05/30 Write block 23 len 24576 data: 9 9 6 6 5 5 06/30 Read block 489 len 8192 07/30 Read block 87 len 16384 08/30 Read block 340 len 20480 09/30 Read block 391 len 28672 10/30 Read block 131 len 4096 11/30 Read block 252 len 16384 12/30 Write block 226 len 36864 data: 8 6 6 6 7 7 8 8 8 13/30 Write block 420 len 36864 data: 5 7 7 6 6 6 7 4 5 14/30 Read block 105 len 12288 15/30 Read block 513 len 40960 16/30 Read block 436 len 12288 17/30 Read block 595 len 12288 18/30 Read block 430 len 24576 19/30 Read block 432 len 16384 20/30 Read block 115 len 40960 21/30 Read block 153 len 32768 22/30 Read block 273 len 16384 23/30 Write block 201 len 16384 data: 7 6 7 7 24/30 Write block 138 len 4096 data: 11 25/30 Write block 338 len 8192 data: 5 5 26/30 Write block 67 len 32768 data: 5 7 9 9 7 7 8 8 27/30 Read block 162 len 40960 28/30 Read block 253 len 16384 29/30 Write block 444 len 20480 data: 7 7 9 5 5 30/30 Read block 173 len 12288 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:10.93478411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 68 68 68 117 117 117 T T T 2 040-059 68 68 68 117 117 117 F F F 3 060-079 74 74 74 125 125 125 T T T 4 080-099 75 75 75 126 126 126 F F F 5 100-119 75 75 75 126 126 126 T T T 6 120-139 75 75 75 126 126 126 T T T 7 140-159 75 75 75 126 126 126 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 75 75 75 126 126 126 T T T 11 220-239 69 69 69 118 118 118 T T T 12 240-259 73 73 73 124 124 124 F F F 13 260-279 75 75 75 126 126 126 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 64 64 64 111 111 111 T T T 17 340-359 73 73 73 124 124 124 F F F 18 360-379 74 74 74 125 125 125 F F F 19 380-399 71 71 71 121 121 121 T T T 20 400-419 74 74 74 125 125 125 T T T 21 420-439 70 70 70 120 120 120 T T T 22 440-459 73 73 73 124 124 124 T T T 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 F F F 28 560-579 75 75 75 126 126 126 F F F 29 580-599 75 75 75 126 126 126 F F F Max gen: 75, Max flush: 126 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 118 118 118 F F F 1 020-039 76 76 76 127 127 127 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 76 76 76 127 127 127 F F F 6 120-139 76 76 76 127 127 127 F F F 7 140-159 75 75 75 126 126 126 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 76 76 76 127 127 127 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 75 75 75 126 126 126 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 64 64 64 111 111 111 F F F 16 320-339 76 76 76 127 127 127 F F F 17 340-359 73 73 73 124 124 124 F F F 18 360-379 74 74 74 125 125 125 F F F 19 380-399 76 76 76 127 127 127 F F F 20 400-419 76 76 76 127 127 127 F F F 21 420-439 76 76 76 127 127 127 F F F 22 440-459 76 76 76 127 127 127 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 F F F 28 560-579 75 75 75 126 126 126 F F F 29 580-599 75 75 75 126 126 126 F F F Max gen: 76, Max flush: 127 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.23759758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:15.280864332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.281239464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.313497157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.31406343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314099588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314113647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314128437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} {"msg":"Crucible b63073a3-88c2-4a35-95d7-d60ee0e83873 has session id: 9d7835eb-4e9d-472b-b357-3d993b2c507b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314537856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Upstairs opts: Upstairs UUID: b63073a3-88c2-4a35-95d7-d60ee0e83873, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314583174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Crucible stats registered with UUID: b63073a3-88c2-4a35-95d7-d60ee0e83873","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.314613643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345027747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345053516Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} {"msg":"b63073a3-88c2-4a35-95d7-d60ee0e83873 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345123622Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345143332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.3451652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.34517928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345193569Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345212998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345240197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345266415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345292324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345319563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51869)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345409448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38485)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345451056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61026)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.345644857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.34618952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346218059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346347542Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.34640531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346420559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346872407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.346898295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347064177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347431239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347461167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347476687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347565742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347585241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347602531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347641029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347658548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[0]R flush_numbers[0..12]: [118, 127, 117, 127, 126, 127, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347683956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[0]R generation[0..12]: [69, 76, 68, 76, 75, 76, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347698196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347712455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[1]R flush_numbers[0..12]: [118, 127, 117, 127, 126, 127, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347726345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[1]R generation[0..12]: [69, 76, 68, 76, 75, 76, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347739844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347753653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[2]R flush_numbers[0..12]: [118, 127, 117, 127, 126, 127, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347767403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[2]R generation[0..12]: [69, 76, 68, 76, 75, 76, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347780882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347794651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Max found gen is 77","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347807481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Generation requested: 78 >= found:77","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.34782059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347833629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347848959Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"":"downstairs","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:07:15.347862508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347879567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"0","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347892736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"1","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347905826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"client":"2","":"downstairs","session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347919155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347931624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"b63073a3-88c2-4a35-95d7-d60ee0e83873 is now active with session: 9d7835eb-4e9d-472b-b357-3d993b2c507b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347944474Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347956553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494,"session_id":"9d7835eb-4e9d-472b-b357-3d993b2c507b"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:15.347968733Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 338 len 20480 data: 6 6 5 6 6 02/30 Write block 563 len 16384 data: 8 7 7 7 03/30 Write block 8 len 40960 data: 4 5 6 5 3 5 7 6 7 5 04/30 Write block 4 len 40960 data: 6 6 7 7 5 6 7 6 4 6 05/30 Read block 248 len 12288 06/30 Write block 257 len 16384 data: 3 4 3 7 07/30 Read block 209 len 28672 08/30 Read block 180 len 32768 09/30 Read block 390 len 12288 10/30 Write block 243 len 8192 data: 6 7 11/30 Read block 377 len 8192 12/30 Write block 379 len 16384 data: 5 4 5 5 13/30 Read block 127 len 28672 14/30 Read block 66 len 36864 15/30 Write block 306 len 36864 data: 6 8 8 7 7 8 8 10 10 16/30 Read block 551 len 28672 17/30 Read block 4 len 28672 18/30 Read block 231 len 40960 19/30 Write block 397 len 16384 data: 4 4 4 5 20/30 Write block 582 len 8192 data: 8 9 21/30 Read block 356 len 8192 22/30 Write block 256 len 24576 data: 4 4 5 4 8 8 23/30 Write block 247 len 20480 data: 7 6 7 8 7 24/30 Write block 101 len 16384 data: 7 8 8 9 25/30 Write block 338 len 12288 data: 7 7 6 26/30 Flush 27/30 Write block 510 len 32768 data: 6 7 7 7 7 8 8 7 28/30 Write block 384 len 36864 data: 7 8 8 7 5 5 6 6 7 29/30 Write block 31 len 20480 data: 4 5 4 5 9 30/30 Read block 492 len 24576 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:15.36632157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 76 76 76 127 127 127 T T T 2 040-059 68 68 68 117 117 117 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 78 78 78 128 128 128 F F F 6 120-139 76 76 76 127 127 127 F F F 7 140-159 75 75 75 126 126 126 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 76 76 76 127 127 127 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 78 78 78 128 128 128 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 78 78 78 128 128 128 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 78 78 78 128 128 128 T T T 20 400-419 78 78 78 128 128 128 F F F 21 420-439 76 76 76 127 127 127 F F F 22 440-459 76 76 76 127 127 127 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 73 73 73 124 124 124 T T T 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 F F F 28 560-579 78 78 78 128 128 128 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 78, Max flush: 128 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 79 79 79 129 129 129 F F F 2 040-059 68 68 68 117 117 117 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 78 78 78 128 128 128 F F F 6 120-139 76 76 76 127 127 127 F F F 7 140-159 75 75 75 126 126 126 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 76 76 76 127 127 127 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 78 78 78 128 128 128 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 78 78 78 128 128 128 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 79 79 79 129 129 129 F F F 20 400-419 78 78 78 128 128 128 F F F 21 420-439 76 76 76 127 127 127 F F F 22 440-459 76 76 76 127 127 127 F F F 23 460-479 63 63 63 109 109 109 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 79 79 79 129 129 129 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 F F F 28 560-579 78 78 78 128 128 128 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 79, Max flush: 129 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.623120832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.668127649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.668743319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.668779487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.668794826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.668809176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} {"msg":"Crucible d6b191e0-3a6b-47be-8be5-a01573c8c77b has session id: 2e330b27-460c-4697-a5ee-b280adf735cd","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.66932636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Upstairs opts: Upstairs UUID: d6b191e0-3a6b-47be-8be5-a01573c8c77b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.669369948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Crucible stats registered with UUID: d6b191e0-3a6b-47be-8be5-a01573c8c77b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.669394827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699625351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699654759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} {"msg":"d6b191e0-3a6b-47be-8be5-a01573c8c77b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699722636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699742805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699765084Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699779653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699794472Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699807542Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699820911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.69983415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.69984775Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699861239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45959)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.699977403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50858)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.70003916Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63120)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700204412Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700716827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700747506Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700763405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700781244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700795553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.700812942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.701408983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.701439451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.70146589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.701974575Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702014263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702031762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702044802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702058811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.70207267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.7020851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702098559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[0]R flush_numbers[0..12]: [128, 129, 117, 127, 126, 128, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702112668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[0]R generation[0..12]: [78, 79, 68, 76, 75, 78, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702129978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702144647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[1]R flush_numbers[0..12]: [128, 129, 117, 127, 126, 128, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702158676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[1]R generation[0..12]: [78, 79, 68, 76, 75, 78, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702172185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702186065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[2]R flush_numbers[0..12]: [128, 129, 117, 127, 126, 128, 127, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702199734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[2]R generation[0..12]: [78, 79, 68, 76, 75, 78, 76, 75, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702213313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702227093Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702249041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702265701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702290259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702308319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"":"downstairs","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702321748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702333877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"0","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702357896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"1","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702380325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"client":"2","":"downstairs","session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702394485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702406994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"d6b191e0-3a6b-47be-8be5-a01573c8c77b is now active with session: 2e330b27-460c-4697-a5ee-b280adf735cd","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702419823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:19.702440832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502,"session_id":"2e330b27-460c-4697-a5ee-b280adf735cd"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:07:19.702461251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 117 len 32768 data: 7 7 6 6 4 3 4 5 02/30 Read block 86 len 12288 03/30 Write block 471 len 20480 data: 6 4 4 5 4 04/30 Flush 05/30 Write block 465 len 24576 data: 5 6 6 4 6 6 06/30 Read block 391 len 32768 07/30 Write block 131 len 12288 data: 9 7 7 08/30 Write block 310 len 12288 data: 8 9 9 09/30 Read block 248 len 8192 10/30 Write block 300 len 40960 data: 7 7 6 6 5 3 7 9 9 8 11/30 Read block 521 len 4096 12/30 Read block 337 len 24576 13/30 Write block 465 len 32768 data: 6 7 7 5 7 7 7 5 14/30 Write block 50 len 20480 data: 7 6 7 6 8 15/30 Read block 109 len 32768 16/30 Flush 17/30 Flush 18/30 Write block 507 len 24576 data: 3 5 7 7 8 8 19/30 Write block 127 len 4096 data: 7 20/30 Write block 358 len 40960 data: 4 5 5 6 7 5 5 6 7 6 21/30 Write block 558 len 24576 data: 10 9 9 9 8 9 22/30 Read block 189 len 36864 23/30 Read block 98 len 28672 24/30 Write block 142 len 40960 data: 10 11 8 7 5 6 4 3 3 4 25/30 Read block 471 len 40960 26/30 Read block 243 len 36864 27/30 Read block 192 len 40960 28/30 Read block 456 len 36864 29/30 Read block 19 len 16384 30/30 Write block 306 len 8192 data: 8 10 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:19.72241145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 79 79 79 129 129 129 F F F 2 040-059 80 80 80 131 131 131 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 80 80 80 130 130 130 F F F 6 120-139 80 80 80 131 131 131 T T T 7 140-159 75 75 75 126 126 126 T T T 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 76 76 76 127 127 127 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 80 80 80 131 131 131 T T T 16 320-339 78 78 78 128 128 128 F F F 17 340-359 78 78 78 128 128 128 T T T 18 360-379 78 78 78 128 128 128 T T T 19 380-399 79 79 79 129 129 129 F F F 20 400-419 78 78 78 128 128 128 F F F 21 420-439 76 76 76 127 127 127 F F F 22 440-459 76 76 76 127 127 127 F F F 23 460-479 80 80 80 131 131 131 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 79 79 79 129 129 129 T T T 26 520-539 70 70 70 120 120 120 F F F 27 540-559 75 75 75 126 126 126 T T T 28 560-579 78 78 78 128 128 128 T T T 29 580-599 78 78 78 128 128 128 F F F Max gen: 80, Max flush: 131 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 79 79 79 129 129 129 F F F 2 040-059 80 80 80 131 131 131 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 80 80 80 130 130 130 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 81 81 81 132 132 132 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 76 76 76 127 127 127 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 75 75 75 126 126 126 F F F 15 300-319 81 81 81 132 132 132 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 81 81 81 132 132 132 F F F 18 360-379 81 81 81 132 132 132 F F F 19 380-399 79 79 79 129 129 129 F F F 20 400-419 78 78 78 128 128 128 F F F 21 420-439 76 76 76 127 127 127 F F F 22 440-459 76 76 76 127 127 127 F F F 23 460-479 80 80 80 131 131 131 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 70 70 70 120 120 120 F F F 27 540-559 81 81 81 132 132 132 F F F 28 560-579 81 81 81 132 132 132 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 81, Max flush: 132 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.009713412Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.056380038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057003717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057044545Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057059444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057073914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} {"msg":"Crucible 8c142e16-5773-4b0a-9b89-868c2e4ffebc has session id: 7864e65e-4a7e-4511-894e-92e32ae95d4a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057485023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Upstairs opts: Upstairs UUID: 8c142e16-5773-4b0a-9b89-868c2e4ffebc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.057529501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Crucible stats registered with UUID: 8c142e16-5773-4b0a-9b89-868c2e4ffebc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.05755634Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.08784732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087875459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} {"msg":"8c142e16-5773-4b0a-9b89-868c2e4ffebc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087888329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087901248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087923747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087941616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087955835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087969254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.087984214Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088008843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088037011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088085859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58887)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088128677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48895)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088157465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65355)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088381704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.088971895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089004483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089021523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089067371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.0890822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089099449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089616873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089701169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.089781225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090220424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090241813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090256032Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090350057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090372086Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090385996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090398285Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090411644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[0]R flush_numbers[0..12]: [128, 129, 131, 127, 126, 130, 132, 132, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090425694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[0]R generation[0..12]: [78, 79, 80, 76, 75, 80, 81, 81, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090439903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090454192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[1]R flush_numbers[0..12]: [128, 129, 131, 127, 126, 130, 132, 132, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090472231Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[1]R generation[0..12]: [78, 79, 80, 76, 75, 80, 81, 81, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090486421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.09050058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[2]R flush_numbers[0..12]: [128, 129, 131, 127, 126, 130, 132, 132, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090514609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[2]R generation[0..12]: [78, 79, 80, 76, 75, 80, 81, 81, 75, 74, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090528379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090542468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090555527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090568697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090581696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090594705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"":"downstairs","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:07:24.090607855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090630234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"0","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090643413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"1","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090656532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"client":"2","":"downstairs","session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090669862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090683891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"8c142e16-5773-4b0a-9b89-868c2e4ffebc is now active with session: 7864e65e-4a7e-4511-894e-92e32ae95d4a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.09069674Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.09070909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510,"session_id":"7864e65e-4a7e-4511-894e-92e32ae95d4a"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:24.090721629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 229 len 24576 data: 7 8 8 9 9 9 02/30 Read block 210 len 8192 03/30 Read block 225 len 24576 04/30 Write block 375 len 28672 data: 8 8 8 7 6 5 6 05/30 Read block 193 len 8192 06/30 Read block 578 len 40960 07/30 Read block 564 len 24576 08/30 Write block 539 len 16384 data: 5 5 5 6 09/30 Write block 433 len 32768 data: 6 6 6 4 6 7 8 9 10/30 Write block 376 len 12288 data: 9 9 8 11/30 Read block 390 len 8192 12/30 Read block 510 len 4096 13/30 Write block 358 len 12288 data: 5 6 6 14/30 Write block 547 len 20480 data: 8 7 5 4 4 15/30 Flush 16/30 Write block 417 len 20480 data: 8 8 7 6 8 17/30 Write block 542 len 32768 data: 7 5 5 8 8 9 8 6 18/30 Read block 373 len 16384 19/30 Write block 287 len 8192 data: 9 10 20/30 Write block 37 len 40960 data: 10 8 8 9 9 10 11 12 8 6 21/30 Read block 527 len 40960 22/30 Write block 34 len 20480 data: 6 10 10 11 9 23/30 Read block 207 len 20480 24/30 Write block 299 len 40960 data: 7 8 8 7 7 6 4 9 11 10 25/30 Read block 284 len 32768 26/30 Write block 463 len 12288 data: 7 6 7 27/30 Read block 180 len 8192 28/30 Read block 342 len 28672 29/30 Read block 463 len 24576 30/30 Read block 279 len 36864 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 14 15 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:24.110936305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 79 79 79 129 129 129 T T T 2 040-059 80 80 80 131 131 131 T T T 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 80 80 80 130 130 130 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 81 81 81 132 132 132 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 82 82 82 133 133 133 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 75 75 75 126 126 126 T T T 15 300-319 81 81 81 132 132 132 T T T 16 320-339 78 78 78 128 128 128 F F F 17 340-359 82 82 82 133 133 133 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 78 78 78 128 128 128 T T T 21 420-439 82 82 82 133 133 133 T T T 22 440-459 82 82 82 133 133 133 F F F 23 460-479 80 80 80 131 131 131 T T T 24 480-499 73 73 73 124 124 124 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 82 82 82 133 133 133 F F F 27 540-559 82 82 82 133 133 133 T T T 28 560-579 81 81 81 132 132 132 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 82, Max flush: 133 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 128 128 128 F F F 1 020-039 83 83 83 134 134 134 F F F 2 040-059 83 83 83 134 134 134 F F F 3 060-079 76 76 76 127 127 127 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 80 80 80 130 130 130 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 81 81 81 132 132 132 F F F 8 160-179 75 75 75 126 126 126 F F F 9 180-199 74 74 74 125 125 125 F F F 10 200-219 76 76 76 127 127 127 F F F 11 220-239 82 82 82 133 133 133 F F F 12 240-259 78 78 78 128 128 128 F F F 13 260-279 78 78 78 128 128 128 F F F 14 280-299 83 83 83 134 134 134 F F F 15 300-319 83 83 83 134 134 134 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 82 82 82 133 133 133 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 83 83 83 134 134 134 F F F 21 420-439 83 83 83 134 134 134 F F F 22 440-459 82 82 82 133 133 133 F F F 23 460-479 83 83 83 134 134 134 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 82 82 82 133 133 133 F F F 27 540-559 83 83 83 134 134 134 F F F 28 560-579 81 81 81 132 132 132 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 83, Max flush: 134 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.394987298Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.440797245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.441593346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.441637694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.441653713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.441668802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} {"msg":"Crucible c7f205a2-0c04-4017-b582-88d2884bcc77 has session id: eb5120c1-ff02-4144-a60d-daf196e5a56e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.442069633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Upstairs opts: Upstairs UUID: c7f205a2-0c04-4017-b582-88d2884bcc77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.442108391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Crucible stats registered with UUID: c7f205a2-0c04-4017-b582-88d2884bcc77","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.44212443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472474078Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472508936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} {"msg":"c7f205a2-0c04-4017-b582-88d2884bcc77 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472536975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472565233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472612201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472648909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472687837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472785792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472909136Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472955814Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.472988132Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473015891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35535)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.47304237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47744)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473070318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51499)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473096717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473555845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473596603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473626681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473656089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473738456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.473761094Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.47425062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474285698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474440861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474784644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474825862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474856521Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474924737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474953536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474968565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474981444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.474994914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[0]R flush_numbers[0..12]: [128, 134, 134, 127, 126, 130, 132, 132, 126, 125, 127, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475020433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[0]R generation[0..12]: [78, 83, 83, 76, 75, 80, 81, 81, 75, 74, 76, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475039112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475053561Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[1]R flush_numbers[0..12]: [128, 134, 134, 127, 126, 130, 132, 132, 126, 125, 127, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.47506744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[1]R generation[0..12]: [78, 83, 83, 76, 75, 80, 81, 81, 75, 74, 76, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475085329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475099889Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[2]R flush_numbers[0..12]: [128, 134, 134, 127, 126, 130, 132, 132, 126, 125, 127, 133]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475113858Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[2]R generation[0..12]: [78, 83, 83, 76, 75, 80, 81, 81, 75, 74, 76, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475127487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475141327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475154406Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475176715Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475189724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475202684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"":"downstairs","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475215693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475227732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"0","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475240942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"1","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475253881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"client":"2","":"downstairs","session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.47526701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.4752794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"c7f205a2-0c04-4017-b582-88d2884bcc77 is now active with session: eb5120c1-ff02-4144-a60d-daf196e5a56e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475293999Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475306288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518,"session_id":"eb5120c1-ff02-4144-a60d-daf196e5a56e"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:28.475318548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 4096 data: 7 02/30 Write block 212 len 24576 data: 5 6 6 7 9 7 03/30 Write block 477 len 28672 data: 4 4 4 5 8 7 5 04/30 Write block 9 len 36864 data: 7 8 7 5 7 8 7 8 6 05/30 Write block 427 len 20480 data: 5 6 4 4 5 06/30 Write block 273 len 32768 data: 7 8 6 6 7 4 3 4 07/30 Read block 151 len 16384 08/30 Read block 227 len 12288 09/30 Write block 439 len 20480 data: 9 10 8 9 6 10/30 Write block 269 len 40960 data: 8 8 6 7 8 9 7 7 8 5 11/30 Write block 150 len 24576 data: 4 5 3 3 5 7 12/30 Flush 13/30 Write block 237 len 40960 data: 10 7 6 4 6 7 7 8 6 7 14/30 Write block 247 len 4096 data: 8 15/30 Read block 213 len 12288 16/30 Write block 487 len 28672 data: 4 5 5 6 7 7 8 17/30 Write block 238 len 28672 data: 8 7 5 7 8 8 9 18/30 Read block 343 len 12288 19/30 Write block 435 len 24576 data: 7 5 7 8 10 11 20/30 Write block 566 len 24576 data: 8 7 7 8 8 8 21/30 Write block 109 len 16384 data: 11 11 12 11 22/30 Write block 417 len 4096 data: 9 23/30 Write block 168 len 28672 data: 8 8 9 8 10 10 10 24/30 Write block 341 len 40960 data: 7 7 7 5 8 7 6 8 9 7 25/30 Read block 299 len 36864 26/30 Flush 27/30 Write block 55 len 24576 data: 6 7 6 7 6 5 28/30 Write block 5 len 40960 data: 7 8 8 6 8 9 8 6 8 9 29/30 Read block 592 len 12288 30/30 Read block 77 len 16384 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:28.494049167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 135 135 135 T T T 1 020-039 83 83 83 134 134 134 F F F 2 040-059 83 83 83 134 134 134 T T T 3 060-079 76 76 76 127 127 127 T T T 4 080-099 75 75 75 126 126 126 F F F 5 100-119 84 84 84 136 136 136 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 84 84 84 135 135 135 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 84 84 84 135 135 135 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 84 84 84 136 136 136 F F F 13 260-279 84 84 84 135 135 135 F F F 14 280-299 84 84 84 135 135 135 F F F 15 300-319 83 83 83 134 134 134 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 84 84 84 136 136 136 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 82 82 82 133 133 133 F F F 27 540-559 83 83 83 134 134 134 F F F 28 560-579 84 84 84 136 136 136 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 84, Max flush: 136 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 137 137 137 F F F 1 020-039 83 83 83 134 134 134 F F F 2 040-059 85 85 85 137 137 137 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 84 84 84 136 136 136 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 84 84 84 135 135 135 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 84 84 84 135 135 135 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 84 84 84 136 136 136 F F F 13 260-279 84 84 84 135 135 135 F F F 14 280-299 84 84 84 135 135 135 F F F 15 300-319 83 83 83 134 134 134 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 84 84 84 136 136 136 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 82 82 82 133 133 133 F F F 27 540-559 83 83 83 134 134 134 F F F 28 560-579 84 84 84 136 136 136 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 85, Max flush: 137 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.755722609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.802805904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.803530158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.803568106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.803585856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.803601115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} {"msg":"Crucible 54fafcd2-b1e5-4549-b537-b92c512f81de has session id: 45639834-f05d-4414-a315-3ebe688b22ff","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.804015284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Upstairs opts: Upstairs UUID: 54fafcd2-b1e5-4549-b537-b92c512f81de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.804060382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Crucible stats registered with UUID: 54fafcd2-b1e5-4549-b537-b92c512f81de","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.804078121Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835114125Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835140354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} {"msg":"54fafcd2-b1e5-4549-b537-b92c512f81de active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835196421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.8352252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835249919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835267088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835287687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835302006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835315896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835333575Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835348134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835362003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:41387)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835536485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:53147)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835678848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:52298)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.835726235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836211921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.83625047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836281358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836312647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836339915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836368024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836870109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.836963355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837038271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837528197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837552646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837566925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837579314Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837592933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837606243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837618682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837632242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[0]R flush_numbers[0..12]: [137, 134, 137, 137, 126, 136, 132, 135, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837646301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[0]R generation[0..12]: [85, 83, 85, 85, 75, 84, 81, 84, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.8376601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837674309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[1]R flush_numbers[0..12]: [137, 134, 137, 137, 126, 136, 132, 135, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837688169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[1]R generation[0..12]: [85, 83, 85, 85, 75, 84, 81, 84, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837701678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837715607Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[2]R flush_numbers[0..12]: [137, 134, 137, 137, 126, 136, 132, 135, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837729367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[2]R generation[0..12]: [85, 83, 85, 85, 75, 84, 81, 84, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837745956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837760145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837773185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837786394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"}Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837811503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837824432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"":"downstairs","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837837552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837849771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"0","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.83786302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"1","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.83787623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"client":"2","":"downstairs","session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837889529Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837901778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"54fafcd2-b1e5-4549-b537-b92c512f81de is now active with session: 45639834-f05d-4414-a315-3ebe688b22ff","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837914628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837926657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526,"session_id":"45639834-f05d-4414-a315-3ebe688b22ff"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:32.837938847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 20480 data: 8 9 9 10 7 02/30 Write block 402 len 12288 data: 7 7 7 03/30 Write block 108 len 28672 data: 9 12 12 13 12 9 7 04/30 Flush 05/30 Write block 14 len 4096 data: 10 06/30 Read block 485 len 8192 07/30 Write block 502 len 4096 data: 4 08/30 Write block 497 len 28672 data: 9 8 7 7 6 5 3 09/30 Write block 142 len 12288 data: 11 12 9 10/30 Read block 259 len 20480 11/30 Read block 24 len 32768 12/30 Write block 308 len 32768 data: 11 9 9 10 10 11 11 10 13/30 Write block 13 len 24576 data: 9 11 8 9 7 5 14/30 Write block 310 len 24576 data: 10 11 11 12 12 11 15/30 Write block 573 len 28672 data: 9 12 10 9 10 9 8 16/30 Read block 453 len 20480 17/30 Read block 101 len 36864 18/30 Write block 31 len 32768 data: 5 6 5 7 11 11 12 10 19/30 Flush 20/30 Write block 534 len 24576 data: 6 8 8 7 4 6 21/30 Read block 593 len 12288 22/30 Read block 518 len 32768 23/30 Write block 565 len 40960 data: 8 9 8 8 9 9 9 9 10 13 24/30 Read block 47 len 4096 25/30 Read block 493 len 12288 26/30 Read block 536 len 32768 27/30 Write block 263 len 12288 data: 8 8 7 28/30 Read block 424 len 28672 29/30 Read block 50 len 20480 30/30 Write block 284 len 36864 data: 6 6 6 10 11 9 9 6 6 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:32.857055547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 139 139 139 F F F 1 020-039 86 86 86 139 139 139 F F F 2 040-059 85 85 85 137 137 137 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 86 86 86 138 138 138 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 84 84 84 135 135 135 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 84 84 84 135 135 135 T T T 14 280-299 84 84 84 135 135 135 T T T 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 82 82 82 133 133 133 T T T 27 540-559 83 83 83 134 134 134 F F F 28 560-579 86 86 86 139 139 139 T T T 29 580-599 78 78 78 128 128 128 F F F Max gen: 86, Max flush: 139 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 139 139 139 F F F 1 020-039 86 86 86 139 139 139 F F F 2 040-059 85 85 85 137 137 137 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 86 86 86 138 138 138 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 84 84 84 135 135 135 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 87 87 87 140 140 140 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 82 82 82 133 133 133 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 83 83 83 134 134 134 F F F 28 560-579 87 87 87 140 140 140 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 87, Max flush: 140 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.123378901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:37.167102651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.167480202Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.199979654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.200554036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.200590414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.200608333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.200623062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} {"msg":"Crucible 072aba61-d961-4163-b4cf-360b1ba76bb9 has session id: c3e702e5-40d4-48af-9aa0-c99e34a2927c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.201060501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Upstairs opts: Upstairs UUID: 072aba61-d961-4163-b4cf-360b1ba76bb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.201103239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Crucible stats registered with UUID: 072aba61-d961-4163-b4cf-360b1ba76bb9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.201127668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231473885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231518123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} {"msg":"072aba61-d961-4163-b4cf-360b1ba76bb9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231538342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231554692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.23158203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231599919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231616898Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231643057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231677146Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231716254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231764941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231803219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46110)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231833508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37151)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.231863946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52722)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.232183421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.232635928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.232683256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.232727424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.232763992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.233090326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.233135234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.233428969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.233604141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.233924445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234078377Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234103426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234118606Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234250109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234269948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234283758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234400922Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234420701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 137, 137, 126, 138, 132, 139, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.23444102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[0]R generation[0..12]: [86, 86, 85, 85, 75, 86, 81, 86, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234455359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234469428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 137, 137, 126, 138, 132, 139, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234483258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[1]R generation[0..12]: [86, 86, 85, 85, 75, 86, 81, 86, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234496617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234510376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 137, 137, 126, 138, 132, 139, 136, 135, 135, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234524036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[2]R generation[0..12]: [86, 86, 85, 85, 75, 86, 81, 86, 84, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234541025Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234555374Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234569173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234582423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234595352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234608522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"":"downstairs","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234621771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.23464154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"0","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234654819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"1","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234667868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"client":"2","":"downstairs","session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234681088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234693387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"072aba61-d961-4163-b4cf-360b1ba76bb9 is now active with session: c3e702e5-40d4-48af-9aa0-c99e34a2927c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234706197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234718186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534,"session_id":"c3e702e5-40d4-48af-9aa0-c99e34a2927c"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:37.234730525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 358 len 12288 data: 6 7 7 02/30 Read block 545 len 36864 03/30 Read block 546 len 28672 04/30 Write block 563 len 36864 data: 10 8 9 10 9 9 10 10 10 05/30 Write block 202 len 28672 data: 7 8 8 6 5 5 3 06/30 Read block 32 len 24576 07/30 Read block 295 len 8192 08/30 Write block 87 len 16384 data: 5 8 8 9 09/30 Write block 42 len 32768 data: 11 12 13 9 7 7 7 6 10/30 Flush 11/30 Write block 550 len 4096 data: 5 12/30 Read block 79 len 20480 13/30 Flush 14/30 Write block 17 len 32768 data: 8 6 6 7 10 9 10 10 15/30 Write block 351 len 12288 data: 8 9 10 16/30 Flush 17/30 Write block 127 len 40960 data: 8 7 9 10 10 8 8 8 7 8 18/30 Read block 428 len 28672 19/30 Read block 462 len 24576 20/30 Read block 53 len 36864 21/30 Write block 103 len 40960 data: 9 10 12 10 10 10 13 13 14 13 22/30 Read block 300 len 28672 23/30 Write block 133 len 28672 data: 9 9 8 9 9 12 10 24/30 Write block 295 len 20480 data: 9 9 11 8 8 25/30 Read block 548 len 12288 26/30 Flush 27/30 Write block 16 len 20480 data: 10 9 7 7 8 28/30 Write block 115 len 4096 data: 8 29/30 Read block 416 len 36864 30/30 Read block 163 len 8192 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:37.254953141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 143 143 143 T T T 1 020-039 88 88 88 143 143 143 T T T 2 040-059 88 88 88 141 141 141 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 88 88 88 141 141 141 F F F 5 100-119 88 88 88 144 144 144 T T T 6 120-139 88 88 88 144 144 144 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 88 88 88 144 144 144 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 88 88 88 142 142 142 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 88, Max flush: 144 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 145 145 145 F F F 1 020-039 89 89 89 145 145 145 F F F 2 040-059 88 88 88 141 141 141 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 88 88 88 141 141 141 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 88 88 88 144 144 144 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 84 84 84 136 136 136 F F F 9 180-199 84 84 84 135 135 135 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 88 88 88 144 144 144 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 88 88 88 141 141 141 F F F 19 380-399 82 82 82 133 133 133 F F F 20 400-419 86 86 86 138 138 138 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 88 88 88 142 142 142 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 89, Max flush: 145 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.547354753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.592080344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.592690584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.592727133Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.592741602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.592756531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} {"msg":"Crucible 2b8138a0-f9f5-430d-8121-85e29af96fbe has session id: 3892144e-7b31-4ff9-bf49-96895badbbf9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.593228328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Upstairs opts: Upstairs UUID: 2b8138a0-f9f5-430d-8121-85e29af96fbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.593275696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Crucible stats registered with UUID: 2b8138a0-f9f5-430d-8121-85e29af96fbe","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.593293725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623535178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623568126Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} {"msg":"2b8138a0-f9f5-430d-8121-85e29af96fbe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623602844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623619883Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623646632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623661352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623675551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62368912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62370283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623720719Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623747377Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623775046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53318)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623802245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58118)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623832503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38413)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.623865682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.624604525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.624635554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.624651853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.624666272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.624680271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62470031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625355628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625395196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625426475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625943309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625973638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.625988577Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626001166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626014736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626031765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626044394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626057694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 141, 137, 141, 145, 144, 139, 136, 135, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626071533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[0]R generation[0..12]: [89, 89, 88, 85, 88, 89, 88, 86, 84, 84, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626085402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626099622Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 141, 137, 141, 145, 144, 139, 136, 135, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626113591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[1]R generation[0..12]: [89, 89, 88, 85, 88, 89, 88, 86, 84, 84, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62612723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62614143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 141, 137, 141, 145, 144, 139, 136, 135, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626155399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[2]R generation[0..12]: [89, 89, 88, 85, 88, 89, 88, 86, 84, 84, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626169258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626184968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626208976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626224046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Next flush: 146","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-26T22:07:41.626248844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626262564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"":"downstairs","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626275953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626288152Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"0","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626301342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"1","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626314421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"client":"2","":"downstairs","session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626327781Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.62634011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"2b8138a0-f9f5-430d-8121-85e29af96fbe is now active with session: 3892144e-7b31-4ff9-bf49-96895badbbf9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626353819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626365928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542,"session_id":"3892144e-7b31-4ff9-bf49-96895badbbf9"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:41.626378128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 376 len 40960 data: 10 10 9 7 6 7 6 6 8 9 03/30 Read block 52 len 40960 04/30 Read block 139 len 8192 05/30 Write block 436 len 40960 data: 6 8 9 11 12 9 10 7 8 8 06/30 Write block 41 len 20480 data: 10 12 13 14 10 07/30 Write block 362 len 8192 data: 8 6 08/30 Write block 196 len 4096 data: 9 09/30 Write block 432 len 40960 data: 6 7 7 8 7 9 10 12 13 10 10/30 Write block 440 len 20480 data: 14 11 11 8 9 11/30 Flush 12/30 Write block 91 len 12288 data: 9 9 7 13/30 Flush 14/30 Write block 409 len 24576 data: 10 6 5 6 5 9 15/30 Read block 72 len 8192 16/30 Read block 80 len 4096 17/30 Flush 18/30 Flush 19/30 Write block 430 len 12288 data: 5 6 7 20/30 Write block 164 len 8192 data: 7 6 21/30 Read block 183 len 32768 22/30 Read block 362 len 20480 23/30 Flush 24/30 Flush 25/30 Write block 554 len 8192 data: 3 3 26/30 Flush 27/30 Write block 418 len 20480 data: 9 8 7 9 8 28/30 Write block 406 len 16384 data: 6 8 8 11 29/30 Write block 444 len 16384 data: 10 9 10 6 30/30 Read block 364 len 24576 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 2 4 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:41.643327005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 145 145 145 F F F 1 020-039 89 89 89 145 145 145 F F F 2 040-059 90 90 90 147 147 147 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 90 90 90 148 148 148 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 88 88 88 144 144 144 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 90 90 90 147 147 147 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 88 88 88 144 144 144 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 90 90 90 149 149 149 T T T 21 420-439 90 90 90 151 151 151 T T T 22 440-459 90 90 90 147 147 147 T T T 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 90, Max flush: 153 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 145 145 145 F F F 1 020-039 89 89 89 145 145 145 F F F 2 040-059 90 90 90 147 147 147 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 90 90 90 148 148 148 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 88 88 88 144 144 144 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 90 90 90 151 151 151 F F F 9 180-199 90 90 90 147 147 147 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 88 88 88 144 144 144 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 91 91 91 154 154 154 F F F 21 420-439 91 91 91 154 154 154 F F F 22 440-459 91 91 91 154 154 154 F F F 23 460-479 84 84 84 135 135 135 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 91, Max flush: 154 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.901920308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:45.94581407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.94621588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.978677303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979272344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979306573Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979320522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979335141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} {"msg":"Crucible 64eee4b7-59b3-4e66-9876-d3fa27466a8e has session id: 5dc78bf0-2fcb-4d89-95af-9fe8470ab949","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979793399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Upstairs opts: Upstairs UUID: 64eee4b7-59b3-4e66-9876-d3fa27466a8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979836617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Crucible stats registered with UUID: 64eee4b7-59b3-4e66-9876-d3fa27466a8e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:45.979861325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010078579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010108728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} {"msg":"64eee4b7-59b3-4e66-9876-d3fa27466a8e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010177455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010196614Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010219593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010233742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010247371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.01026552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010291179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010342157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010374945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010403074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_connection connected from Ok(127.0.0.1:54824)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010433042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_connection connected from Ok(127.0.0.1:48232)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010462041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_connection connected from Ok(127.0.0.1:42301)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.010489659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011238873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011271451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011352037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011370786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011385295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011398995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.011920759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012086011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012128029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012536649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012568077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012584046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012596816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012610355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012623605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012650083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012664363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 147, 137, 148, 145, 144, 139, 151, 147, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.01270863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[0]R generation[0..12]: [89, 89, 90, 85, 90, 89, 88, 86, 90, 90, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012727389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012741809Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 147, 137, 148, 145, 144, 139, 151, 147, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012755698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[1]R generation[0..12]: [89, 89, 90, 85, 90, 89, 88, 86, 90, 90, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012769527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012783517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 147, 137, 148, 145, 144, 139, 151, 147, 141, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012797296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[2]R generation[0..12]: [89, 89, 90, 85, 90, 89, 88, 86, 90, 90, 88, 84]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012810965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012824795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012837844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012851173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012864373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012877272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH"Wait for a query_work_queue command to finish before sending IO ,"pid":1550,"":"downstairs","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012899241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.01291402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"0","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012928799Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"1","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012942129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"client":"2","":"downstairs","session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012955468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012967857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"64eee4b7-59b3-4e66-9876-d3fa27466a8e is now active with session: 5dc78bf0-2fcb-4d89-95af-9fe8470ab949","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012980717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.012992666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550,"session_id":"5dc78bf0-2fcb-4d89-95af-9fe8470ab949"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:46.013005056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 40960 data: 13 15 12 12 9 11 10 11 7 6 02/30 Write block 94 len 12288 data: 6 7 8 03/30 Read block 8 len 24576 04/30 Read block 519 len 32768 05/30 Write block 211 len 12288 data: 6 6 7 06/30 Write block 466 len 32768 data: 8 8 6 8 8 8 6 5 07/30 Write block 234 len 12288 data: 10 10 10 08/30 Flush 09/30 Write block 503 len 20480 data: 4 4 5 3 4 10/30 Write block 284 len 28672 data: 7 7 7 11 12 10 10 11/30 Write block 218 len 4096 data: 9 12/30 Read block 368 len 16384 13/30 Read block 381 len 16384 14/30 Read block 264 len 40960 15/30 Read block 94 len 24576 16/30 Write block 125 len 36864 data: 5 6 9 8 10 11 11 9 10 17/30 Read block 595 len 8192 18/30 Read block 527 len 36864 19/30 Write block 166 len 24576 data: 7 7 9 9 10 9 20/30 Read block 337 len 28672 21/30 Write block 183 len 24576 data: 7 8 9 6 4 4 22/30 Read block 113 len 20480 23/30 Write block 460 len 32768 data: 8 7 6 8 7 8 9 9 24/30 Write block 30 len 4096 data: 6 25/30 Read block 395 len 36864 26/30 Read block 348 len 12288 27/30 Read block 417 len 20480 28/30 Read block 38 len 28672 29/30 Write block 8 len 8192 data: 7 9 30/30 Read block 433 len 12288 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:46.032721886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 145 145 145 T T T 1 020-039 89 89 89 145 145 145 T T T 2 040-059 90 90 90 147 147 147 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 92 92 92 155 155 155 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 88 88 88 144 144 144 T T T 7 140-159 86 86 86 139 139 139 F F F 8 160-179 90 90 90 151 151 151 T T T 9 180-199 90 90 90 147 147 147 T T T 10 200-219 92 92 92 155 155 155 T T T 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 88 88 88 144 144 144 T T T 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 91 91 91 154 154 154 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 92 92 92 155 155 155 F F F 23 460-479 92 92 92 155 155 155 T T T 24 480-499 86 86 86 139 139 139 F F F 25 500-519 86 86 86 139 139 139 T T T 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 92, Max flush: 155 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 F F F 1 020-039 93 93 93 156 156 156 F F F 2 040-059 90 90 90 147 147 147 F F F 3 060-079 85 85 85 137 137 137 F F F 4 080-099 92 92 92 155 155 155 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 93 93 93 156 156 156 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 93 93 93 156 156 156 F F F 10 200-219 93 93 93 156 156 156 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 93 93 93 156 156 156 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 91 91 91 154 154 154 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 92 92 92 155 155 155 F F F 23 460-479 93 93 93 156 156 156 F F F 24 480-499 86 86 86 139 139 139 F F F 25 500-519 93 93 93 156 156 156 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 78 78 78 128 128 128 F F F Max gen: 93, Max flush: 156 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.358212751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.403604909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404220198Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404258787Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404273486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404288135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} {"msg":"Crucible 24496694-add4-4292-b786-c715d94b1be0 has session id: 5444b9f0-5c14-4b85-bff5-c28a5a9c915c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404688475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Upstairs opts: Upstairs UUID: 24496694-add4-4292-b786-c715d94b1be0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404729404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Crucible stats registered with UUID: 24496694-add4-4292-b786-c715d94b1be0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.404746692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435242733Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435270342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} {"msg":"24496694-add4-4292-b786-c715d94b1be0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435342288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435361337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435383846Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435398165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435411995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435425644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435439343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435452903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435466902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435485201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34506)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435688201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57330)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435806755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44928)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.435854373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436279982Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436332499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436355328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436381207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436537049Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436568468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.436895672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437034835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437370208Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437529091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43754843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437562439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437711402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437730441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43774431Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437850595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437869974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[0]R flush_numbers[0..12]: [156, 156, 147, 137, 155, 145, 156, 139, 156, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437888223Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[0]R generation[0..12]: [93, 93, 90, 85, 92, 89, 93, 86, 93, 93, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437902592Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437916802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[1]R flush_numbers[0..12]: [156, 156, 147, 137, 155, 145, 156, 139, 156, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437930641Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[1]R generation[0..12]: [93, 93, 90, 85, 92, 89, 93, 86, 93, 93, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43794425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43795835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[2]R flush_numbers[0..12]: [156, 156, 147, 137, 155, 145, 156, 139, 156, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437972089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[2]R generation[0..12]: [93, 93, 90, 85, 92, 89, 93, 86, 93, 93, 93, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437985748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.437999788Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438013037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438026346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438039266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438052135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"":"downstairs","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438065704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438079364Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"0","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:07:50.438092903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"1","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438113652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"client":"2","":"downstairs","session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438127251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438139671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"24496694-add4-4292-b786-c715d94b1be0 is now active with session: 5444b9f0-5c14-4b85-bff5-c28a5a9c915c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43815245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.43816452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558,"session_id":"5444b9f0-5c14-4b85-bff5-c28a5a9c915c"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:50.438176779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 32768 data: 7 8 10 8 7 7 7 8 02/30 Write block 478 len 40960 data: 5 5 6 9 8 6 5 4 4 5 03/30 Flush 04/30 Write block 38 len 8192 data: 11 9 05/30 Write block 58 len 24576 data: 8 7 6 5 8 9 06/30 Read block 573 len 40960 07/30 Read block 380 len 24576 08/30 Write block 400 len 4096 data: 6 09/30 Read block 562 len 4096 10/30 Write block 79 len 12288 data: 3 2 3 11/30 Flush 12/30 Write block 496 len 4096 data: 7 13/30 Write block 452 len 4096 data: 2 14/30 Flush 15/30 Write block 44 len 28672 data: 15 11 8 8 8 7 8 16/30 Write block 498 len 24576 data: 9 8 8 7 6 5 17/30 Write block 477 len 40960 data: 5 6 6 7 10 9 7 6 5 5 18/30 Read block 98 len 8192 19/30 Write block 597 len 8192 data: 5 4 20/30 Read block 271 len 8192 21/30 Read block 454 len 16384 22/30 Read block 106 len 16384 23/30 Flush 24/30 Flush 25/30 Write block 342 len 16384 data: 8 8 6 9 26/30 Write block 185 len 32768 data: 10 7 5 5 4 4 6 5 27/30 Read block 163 len 12288 28/30 Write block 40 len 32768 data: 10 11 13 14 16 12 9 9 29/30 Read block 583 len 16384 30/30 Read block 298 len 24576 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:50.456179344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 F F F 1 020-039 94 94 94 158 158 158 F F F 2 040-059 94 94 94 160 160 160 T T T 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 93 93 93 156 156 156 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 94 94 94 157 157 157 T T T 10 200-219 94 94 94 157 157 157 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 93 93 93 156 156 156 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 88 88 88 143 143 143 T T T 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 94 94 94 158 158 158 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 94 94 94 160 160 160 F F F Max gen: 94, Max flush: 160 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 F F F 1 020-039 94 94 94 158 158 158 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 93 93 93 156 156 156 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 95 95 95 161 161 161 F F F 10 200-219 94 94 94 157 157 157 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 93 93 93 156 156 156 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 78 78 78 128 128 128 F F F 17 340-359 95 95 95 161 161 161 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 90 90 90 147 147 147 F F F 20 400-419 94 94 94 158 158 158 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 88 88 88 141 141 141 F F F 29 580-599 94 94 94 160 160 160 F F F Max gen: 95, Max flush: 161 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.717254955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.762258912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763216235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763253393Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763267892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763282762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} {"msg":"Crucible 67e96928-2995-4e4d-a673-a5709d341293 has session id: 44f7cba1-03f4-4992-8591-d17aeec55383","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763780037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Upstairs opts: Upstairs UUID: 67e96928-2995-4e4d-a673-a5709d341293, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763827865Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Crucible stats registered with UUID: 67e96928-2995-4e4d-a673-a5709d341293","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.763852234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.795936626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.795966354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} {"msg":"67e96928-2995-4e4d-a673-a5709d341293 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796028022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.79605762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796088159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796111697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796132976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796153815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796169455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796182774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796196623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796219462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_connection connected from Ok(127.0.0.1:53651)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.79625244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_connection connected from Ok(127.0.0.1:49711)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796275699Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_connection connected from Ok(127.0.0.1:49859)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.796304568Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797137377Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797167666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797183745Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797198234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797252501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797268161Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797822303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797960416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.797991035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798351467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798380726Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798395835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798573666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798602245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798617234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798629883Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798643443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[0]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 156, 139, 156, 161, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798657442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[0]R generation[0..12]: [93, 94, 95, 94, 94, 89, 93, 86, 93, 95, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798671372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798685981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[1]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 156, 139, 156, 161, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.79870404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[1]R generation[0..12]: [93, 94, 95, 94, 94, 89, 93, 86, 93, 95, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798718259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798732398Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[2]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 156, 139, 156, 161, 157, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798746278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[2]R generation[0..12]: [93, 94, 95, 94, 94, 89, 93, 86, 93, 95, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798759797Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798773696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798786656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:07:54.798799995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.79889901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.79891678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"":"downstairs","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798930219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798942478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"0","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798955827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"1","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798968747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"client":"2","":"downstairs","session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798981976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.798996845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"67e96928-2995-4e4d-a673-a5709d341293 is now active with session: 44f7cba1-03f4-4992-8591-d17aeec55383","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.799009855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.799021794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566,"session_id":"44f7cba1-03f4-4992-8591-d17aeec55383"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:54.799034134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 486 len 4096 data: 6 02/30 Read block 267 len 40960 03/30 Write block 292 len 20480 data: 7 5 7 10 10 04/30 Write block 382 len 4096 data: 7 05/30 Read block 93 len 20480 06/30 Write block 192 len 40960 data: 6 7 8 9 11 9 8 8 8 9 07/30 Read block 129 len 28672 08/30 Read block 176 len 28672 09/30 Read block 400 len 16384 10/30 Flush 11/30 Write block 128 len 24576 data: 9 11 12 12 10 11 12/30 Write block 576 len 32768 data: 10 11 10 9 8 9 9 10 13/30 Write block 544 len 20480 data: 6 9 9 10 9 14/30 Read block 218 len 16384 15/30 Write block 286 len 8192 data: 8 12 16/30 Read block 233 len 36864 17/30 Read block 91 len 36864 18/30 Read block 371 len 24576 19/30 Read block 518 len 8192 20/30 Read block 172 len 20480 21/30 Flush 22/30 Write block 579 len 4096 data: 10 23/30 Read block 447 len 32768 24/30 Write block 319 len 40960 data: 7 6 7 8 8 10 9 7 7 7 25/30 Write block 487 len 36864 data: 6 6 6 7 8 8 9 5 5 26/30 Read block 121 len 32768 27/30 Read block 296 len 4096 28/30 Write block 348 len 16384 data: 9 10 8 9 29/30 Read block 151 len 12288 30/30 Read block 560 len 36864 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:54.818920926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 F F F 1 020-039 94 94 94 158 158 158 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 96 96 96 163 163 163 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 96 96 96 162 162 162 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 96 96 96 163 163 163 F F F 15 300-319 86 86 86 139 139 139 T T T 16 320-339 78 78 78 128 128 128 T T T 17 340-359 95 95 95 161 161 161 T T T 18 360-379 90 90 90 147 147 147 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 94 94 94 158 158 158 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 96 96 96 162 162 162 T T T 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 96 96 96 163 163 163 F F F 28 560-579 96 96 96 163 163 163 T T T 29 580-599 96 96 96 163 163 163 F F F Max gen: 96, Max flush: 163 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 F F F 1 020-039 94 94 94 158 158 158 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 96 96 96 163 163 163 F F F 7 140-159 86 86 86 139 139 139 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 96 96 96 162 162 162 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 F F F 14 280-299 96 96 96 163 163 163 F F F 15 300-319 97 97 97 164 164 164 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 97 97 97 164 164 164 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 94 94 94 158 158 158 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 94 94 94 160 160 160 F F F 24 480-499 97 97 97 164 164 164 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 96 96 96 163 163 163 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 96 96 96 163 163 163 F F F Max gen: 97, Max flush: 164 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.08667941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.133423812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.1340563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134086649Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134104628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134119107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} {"msg":"Crucible 9254839d-3edc-4d9f-9263-36bdd15f1c82 has session id: 9631f0ef-8fa7-43f5-ad5c-70fd982cc24b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134580225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Upstairs opts: Upstairs UUID: 9254839d-3edc-4d9f-9263-36bdd15f1c82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134624813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Crucible stats registered with UUID: 9254839d-3edc-4d9f-9263-36bdd15f1c82","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.134646981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165053976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165089435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} {"msg":"9254839d-3edc-4d9f-9263-36bdd15f1c82 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165112023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165128553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165156191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.16517439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165192039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165209489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165231008Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165248447Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165278885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165315253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53781)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165349912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62555)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.16538015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65223)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.165613119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166033318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166062877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.16619271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166229628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166256327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166278456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166642548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.166837978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167157963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167293106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167349993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167364803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167377412Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167391031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167404241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167607091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.1676268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[0]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 163, 139, 156, 162, 162, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167642989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[0]R generation[0..12]: [93, 94, 95, 94, 94, 89, 96, 86, 93, 96, 96, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167657398Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167671448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[1]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 163, 139, 156, 162, 162, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167685487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[1]R generation[0..12]: [93, 94, 95, 94, 94, 89, 96, 86, 93, 96, 96, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167699166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167713055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[2]R flush_numbers[0..12]: [156, 158, 161, 158, 158, 145, 163, 139, 156, 162, 162, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167729915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[2]R generation[0..12]: [93, 94, 95, 94, 94, 89, 96, 86, 93, 96, 96, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167743944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167757813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167770793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167784072Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167797071Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167809981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"":"downstairs","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-11-26T22:07:59.1678316Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167843849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"0","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167857079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"1","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167870248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"client":"2","":"downstairs","session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167883527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167895826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"9254839d-3edc-4d9f-9263-36bdd15f1c82 is now active with session: 9631f0ef-8fa7-43f5-ad5c-70fd982cc24b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167908676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167920695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574,"session_id":"9631f0ef-8fa7-43f5-ad5c-70fd982cc24b"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:07:59.167933835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 12288 data: 3 4 7 02/30 Read block 234 len 20480 03/30 Write block 480 len 28672 data: 8 11 10 8 7 6 7 04/30 Read block 301 len 36864 05/30 Write block 138 len 20480 data: 13 11 9 11 12 06/30 Flush 07/30 Write block 586 len 32768 data: 8 8 8 9 9 8 7 6 08/30 Read block 585 len 36864 09/30 Read block 493 len 4096 10/30 Write block 353 len 28672 data: 11 9 7 5 4 7 8 11/30 Read block 444 len 40960 12/30 Write block 130 len 32768 data: 13 13 11 12 10 9 10 10 13/30 Read block 88 len 8192 14/30 Read block 559 len 36864 15/30 Write block 395 len 40960 data: 3 4 5 5 5 7 6 8 8 8 16/30 Read block 497 len 16384 17/30 Read block 398 len 36864 18/30 Write block 11 len 40960 data: 9 7 10 12 9 11 10 8 8 9 19/30 Read block 193 len 32768 20/30 Read block 189 len 4096 21/30 Read block 266 len 16384 22/30 Read block 500 len 28672 23/30 Write block 263 len 4096 data: 9 24/30 Read block 289 len 24576 25/30 Read block 556 len 16384 26/30 Write block 465 len 8192 data: 9 10 27/30 Read block 213 len 4096 28/30 Read block 566 len 36864 29/30 Read block 54 len 28672 30/30 Read block 469 len 36864 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:07:59.189694135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 156 156 156 T T T 1 020-039 94 94 94 158 158 158 T T T 2 040-059 95 95 95 161 161 161 F F F 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 98 98 98 165 165 165 T T T 7 140-159 98 98 98 165 165 165 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 87 87 87 140 140 140 T T T 14 280-299 96 96 96 163 163 163 F F F 15 300-319 97 97 97 164 164 164 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 97 97 97 164 164 164 T T T 18 360-379 90 90 90 147 147 147 F F F 19 380-399 96 96 96 162 162 162 T T T 20 400-419 94 94 94 158 158 158 T T T 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 94 94 94 160 160 160 T T T 24 480-499 98 98 98 165 165 165 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 96 96 96 163 163 163 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 96 96 96 163 163 163 T T T Max gen: 98, Max flush: 165 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 166 166 166 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 94 94 94 158 158 158 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 89 89 89 145 145 145 F F F 6 120-139 99 99 99 166 166 166 F F F 7 140-159 98 98 98 165 165 165 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 99 99 99 166 166 166 F F F 14 280-299 96 96 96 163 163 163 F F F 15 300-319 97 97 97 164 164 164 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 99 99 99 166 166 166 F F F 18 360-379 90 90 90 147 147 147 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 99 99 99 166 166 166 F F F 21 420-439 92 92 92 155 155 155 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 99 99 99 166 166 166 F F F 24 480-499 98 98 98 165 165 165 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 87 87 87 140 140 140 F F F 27 540-559 96 96 96 163 163 163 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 99 99 99 166 166 166 F F F Max gen: 99, Max flush: 166 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.4803772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:03.524939289Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.525344009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.558606784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559185385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559222683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559237053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559251932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} {"msg":"Crucible 85e3fbb9-3896-4c07-bc4a-1e892e2b8650 has session id: c7c0ca6c-3730-4959-9a9e-2b040b80143b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.55969933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Upstairs opts: Upstairs UUID: 85e3fbb9-3896-4c07-bc4a-1e892e2b8650, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559746968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Crucible stats registered with UUID: 85e3fbb9-3896-4c07-bc4a-1e892e2b8650","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.559772736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590686276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590721334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} {"msg":"85e3fbb9-3896-4c07-bc4a-1e892e2b8650 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590746863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590763922Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590790711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.59080874Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590826319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590848138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590865987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590892376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590913515Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590931354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40820)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.590974742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52719)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.59102099Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50792)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.591059698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.591782432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.591812181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.5918281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.591842099Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.591988762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.592087847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.592495457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.592542615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.592670609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593083558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593115217Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593130416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593143135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593156525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593169784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593181833Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593195113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[0]R flush_numbers[0..12]: [166, 166, 161, 158, 158, 145, 166, 165, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593237891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[0]R generation[0..12]: [99, 99, 95, 94, 94, 89, 99, 98, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.59325534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593269709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[1]R flush_numbers[0..12]: [166, 166, 161, 158, 158, 145, 166, 165, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593283788Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[1]R generation[0..12]: [99, 99, 95, 94, 94, 89, 99, 98, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593297478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593314737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[2]R flush_numbers[0..12]: [166, 166, 161, 158, 158, 145, 166, 165, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593329056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[2]R generation[0..12]: [99, 99, 95, 94, 94, 89, 99, 98, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593342645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593356635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593369694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593382954Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593396003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-26T22:08:03.593467059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"":"downstairs","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593486698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593498998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"0","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593512687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"1","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593526176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"client":"2","":"downstairs","session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593539616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593566585Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"85e3fbb9-3896-4c07-bc4a-1e892e2b8650 is now active with session: c7c0ca6c-3730-4959-9a9e-2b040b80143b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593580444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593595313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582,"session_id":"c7c0ca6c-3730-4959-9a9e-2b040b80143b"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:03.593608263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 8192 data: 11 7 02/30 Write block 435 len 32768 data: 9 8 10 11 14 16 13 13 03/30 Flush 04/30 Write block 64 len 12288 data: 9 7 5 05/30 Write block 363 len 24576 data: 7 6 7 8 7 7 06/30 Read block 97 len 4096 07/30 Write block 529 len 40960 data: 7 7 8 9 7 7 9 9 8 5 08/30 Write block 116 len 20480 data: 9 8 8 7 7 09/30 Read block 99 len 20480 10/30 Read block 371 len 20480 11/30 Read block 319 len 28672 12/30 Write block 406 len 20480 data: 7 9 9 12 7 13/30 Read block 101 len 16384 14/30 Flush 15/30 Write block 442 len 28672 data: 14 10 12 11 12 8 7 16/30 Read block 381 len 8192 17/30 Read block 444 len 28672 18/30 Flush 19/30 Write block 469 len 28672 data: 9 9 9 7 6 6 5 20/30 Read block 118 len 16384 21/30 Write block 539 len 16384 data: 7 6 6 8 22/30 Read block 60 len 12288 23/30 Write block 353 len 4096 data: 12 24/30 Write block 288 len 40960 data: 13 11 12 8 8 6 8 11 11 12 25/30 Write block 443 len 24576 data: 11 13 12 13 9 8 26/30 Write block 141 len 28672 data: 12 13 13 10 8 6 7 27/30 Write block 306 len 28672 data: 10 12 12 10 11 12 12 28/30 Write block 277 len 28672 data: 9 6 4 5 4 4 5 29/30 Write block 114 len 4096 data: 8 30/30 Read block 157 len 36864 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:03.61237088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 166 166 166 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 100 100 100 168 168 168 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 100 100 100 168 168 168 T T T 6 120-139 100 100 100 168 168 168 F F F 7 140-159 98 98 98 165 165 165 T T T 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 99 99 99 166 166 166 T T T 14 280-299 100 100 100 167 167 167 T T T 15 300-319 97 97 97 164 164 164 T T T 16 320-339 97 97 97 164 164 164 F F F 17 340-359 99 99 99 166 166 166 T T T 18 360-379 100 100 100 168 168 168 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 100 100 100 168 168 168 F F F 21 420-439 100 100 100 167 167 167 F F F 22 440-459 100 100 100 169 169 169 T T T 23 460-479 99 99 99 166 166 166 T T T 24 480-499 98 98 98 165 165 165 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 100 100 100 168 168 168 T T T 27 540-559 96 96 96 163 163 163 T T T 28 560-579 97 97 97 164 164 164 F F F 29 580-599 99 99 99 166 166 166 F F F Max gen: 100, Max flush: 169 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 166 166 166 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 95 95 95 161 161 161 F F F 3 060-079 100 100 100 168 168 168 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 101 101 101 170 170 170 F F F 6 120-139 100 100 100 168 168 168 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 F F F 13 260-279 101 101 101 170 170 170 F F F 14 280-299 101 101 101 170 170 170 F F F 15 300-319 101 101 101 170 170 170 F F F 16 320-339 97 97 97 164 164 164 F F F 17 340-359 101 101 101 170 170 170 F F F 18 360-379 100 100 100 168 168 168 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 100 100 100 168 168 168 F F F 21 420-439 100 100 100 167 167 167 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 98 98 98 165 165 165 F F F 25 500-519 94 94 94 160 160 160 F F F 26 520-539 101 101 101 170 170 170 F F F 27 540-559 101 101 101 170 170 170 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 99 99 99 166 166 166 F F F Max gen: 101, Max flush: 170 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:07.940416238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:07.983974446Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:07.984356037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.016714216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017316317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017347915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017362474Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017376914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} {"msg":"Crucible e0ef4152-7480-4cbc-b5d0-5fb08ec79dea has session id: 66887289-5645-4d29-8d79-dec4188727e9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017760835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Upstairs opts: Upstairs UUID: e0ef4152-7480-4cbc-b5d0-5fb08ec79dea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017808572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Crucible stats registered with UUID: e0ef4152-7480-4cbc-b5d0-5fb08ec79dea","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.017834011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048208318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048253426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} {"msg":"e0ef4152-7480-4cbc-b5d0-5fb08ec79dea active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048321602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048349751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048375549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048390639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048404748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048422467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048436466Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048449826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048467875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048490814Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56193)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048737752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37055)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048783389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:65331)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.048813408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049486295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049527123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049686625Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049721153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049751902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.049767271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050076446Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050236878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050549173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050716165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050740633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050754812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050847358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050867067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.050880806Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051132944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051158033Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[0]R flush_numbers[0..12]: [166, 166, 161, 168, 158, 170, 168, 170, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051172702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[0]R generation[0..12]: [99, 99, 95, 100, 94, 101, 100, 101, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051186791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.05120095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[1]R flush_numbers[0..12]: [166, 166, 161, 168, 158, 170, 168, 170, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.05121513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[1]R generation[0..12]: [99, 99, 95, 100, 94, 101, 100, 101, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051229149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051247078Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[2]R flush_numbers[0..12]: [166, 166, 161, 168, 158, 170, 168, 170, 156, 162, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051261428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[2]R generation[0..12]: [99, 99, 95, 100, 94, 101, 100, 101, 93, 96, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051274997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051288936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051301966Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051315445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051328614Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051341694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051363753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051376092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"0","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051389401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"1","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051402561Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"client":"2","":"downstairs","session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.05141579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051428289Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"e0ef4152-7480-4cbc-b5d0-5fb08ec79dea is now active with session: 66887289-5645-4d29-8d79-dec4188727e9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051441089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051455638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590,"session_id":"66887289-5645-4d29-8d79-dec4188727e9"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:08.051468157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 9 len 28672 data: 10 10 10 8 11 13 10 03/30 Write block 1 len 32768 data: 3 5 6 7 8 9 9 8 04/30 Flush 05/30 Write block 51 len 16384 data: 7 8 7 9 06/30 Write block 190 len 12288 data: 5 7 7 07/30 Flush 08/30 Write block 546 len 4096 data: 10 09/30 Read block 158 len 36864 10/30 Read block 546 len 24576 11/30 Flush 12/30 Flush 13/30 Write block 276 len 32768 data: 8 10 7 5 6 5 5 6 14/30 Read block 534 len 20480 15/30 Write block 535 len 4096 data: 10 16/30 Write block 256 len 28672 data: 5 5 6 5 9 9 9 17/30 Read block 5 len 16384 18/30 Write block 263 len 28672 data: 10 9 8 7 8 7 9 19/30 Read block 359 len 24576 20/30 Read block 522 len 24576 21/30 Write block 273 len 36864 data: 9 10 8 9 11 8 6 7 6 22/30 Read block 592 len 8192 23/30 Write block 546 len 24576 data: 11 11 10 7 6 5 24/30 Write block 502 len 12288 data: 7 6 5 25/30 Write block 325 len 36864 data: 10 8 8 8 6 8 7 7 8 26/30 Read block 28 len 20480 27/30 Write block 590 len 24576 data: 10 9 8 7 6 6 28/30 Write block 52 len 12288 data: 9 8 10 29/30 Read block 267 len 24576 30/30 Read block 380 len 32768 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:08.070864854Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 102 102 102 173 173 173 T T T 3 060-079 100 100 100 168 168 168 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 101 101 101 170 170 170 F F F 6 120-139 100 100 100 168 168 168 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 86 86 86 138 138 138 T T T 13 260-279 101 101 101 170 170 170 T T T 14 280-299 101 101 101 170 170 170 T T T 15 300-319 101 101 101 170 170 170 F F F 16 320-339 97 97 97 164 164 164 T T T 17 340-359 101 101 101 170 170 170 F F F 18 360-379 100 100 100 168 168 168 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 100 100 100 168 168 168 F F F 21 420-439 100 100 100 167 167 167 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 98 98 98 165 165 165 F F F 25 500-519 94 94 94 160 160 160 T T T 26 520-539 101 101 101 170 170 170 T T T 27 540-559 102 102 102 174 174 174 T T T 28 560-579 97 97 97 164 164 164 F F F 29 580-599 99 99 99 166 166 166 T T T Max gen: 102, Max flush: 174 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 103 103 103 175 175 175 F F F 3 060-079 100 100 100 168 168 168 F F F 4 080-099 94 94 94 158 158 158 F F F 5 100-119 101 101 101 170 170 170 F F F 6 120-139 100 100 100 168 168 168 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 98 98 98 165 165 165 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 103 103 103 175 175 175 F F F 13 260-279 103 103 103 175 175 175 F F F 14 280-299 103 103 103 175 175 175 F F F 15 300-319 101 101 101 170 170 170 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 101 101 101 170 170 170 F F F 18 360-379 100 100 100 168 168 168 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 100 100 100 168 168 168 F F F 21 420-439 100 100 100 167 167 167 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 98 98 98 165 165 165 F F F 25 500-519 103 103 103 175 175 175 F F F 26 520-539 103 103 103 175 175 175 F F F 27 540-559 103 103 103 175 175 175 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 103 103 103 175 175 175 F F F Max gen: 103, Max flush: 175 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.394820704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:12.438740884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.439115796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.471290744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.471884255Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.471920873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.471935242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.471949871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} {"msg":"Crucible 8936fabf-c8a0-4324-84ff-5ef0b2084acb has session id: 15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.472436777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Upstairs opts: Upstairs UUID: 8936fabf-c8a0-4324-84ff-5ef0b2084acb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.472482385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Crucible stats registered with UUID: 8936fabf-c8a0-4324-84ff-5ef0b2084acb","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.472500324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502796385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502831383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} {"msg":"8936fabf-c8a0-4324-84ff-5ef0b2084acb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502882871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.50290295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502930678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502948207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502968526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.502985755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503002465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503018944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503036053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503053062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54113)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503163417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41938)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503283491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52698)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503329429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503944228Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503973527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.503988556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504002075Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504215145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504244924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504671263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504767768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.504798566Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505271113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505300701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505316201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505412636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505448224Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505470273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505483403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505497512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[0]R flush_numbers[0..12]: [172, 166, 175, 168, 158, 170, 168, 170, 156, 173, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505512031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[0]R generation[0..12]: [102, 99, 103, 100, 94, 101, 100, 101, 93, 102, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505526121Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.50554023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[1]R flush_numbers[0..12]: [172, 166, 175, 168, 158, 170, 168, 170, 156, 173, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505554199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[1]R generation[0..12]: [102, 99, 103, 100, 94, 101, 100, 101, 93, 102, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505570938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505585487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[2]R flush_numbers[0..12]: [172, 166, 175, 168, 158, 170, 168, 170, 156, 173, 165, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505599307Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[2]R generation[0..12]: [102, 99, 103, 100, 94, 101, 100, 101, 93, 102, 98, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505612956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505631255Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:08:12.505656344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505683693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505703752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505717441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"":"downstairs","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.5057306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.50574269Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"0","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505756159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"1","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505769549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"client":"2","":"downstairs","session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505783148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505795697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"8936fabf-c8a0-4324-84ff-5ef0b2084acb is now active with session: 15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505811227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505823526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598,"session_id":"15c9bbfa-3057-4ab6-b7b7-3ae7f12ddbfc"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:12.505835955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 36864 data: 6 9 10 10 11 8 8 9 7 02/30 Read block 463 len 8192 03/30 Write block 541 len 32768 data: 7 9 6 7 10 12 12 11 04/30 Flush 05/30 Write block 274 len 40960 data: 11 9 10 12 9 7 8 7 6 7 06/30 Write block 371 len 24576 data: 9 8 8 7 9 11 07/30 Write block 78 len 28672 data: 3 4 3 4 4 5 6 08/30 Write block 95 len 40960 data: 8 9 6 6 5 5 8 9 10 11 09/30 Read block 406 len 8192 10/30 Read block 39 len 20480 11/30 Read block 63 len 40960 12/30 Write block 497 len 40960 data: 10 10 9 9 8 8 7 6 6 4 13/30 Write block 213 len 16384 data: 8 7 8 10 14/30 Read block 513 len 16384 15/30 Flush 16/30 Write block 112 len 32768 data: 14 10 9 9 10 9 9 8 17/30 Flush 18/30 Write block 209 len 20480 data: 4 5 8 7 9 19/30 Write block 417 len 28672 data: 10 10 9 8 10 9 7 20/30 Write block 204 len 28672 data: 9 7 6 6 4 5 6 21/30 Write block 411 len 32768 data: 6 7 6 10 9 10 11 11 22/30 Read block 487 len 36864 23/30 Write block 295 len 40960 data: 12 12 13 9 9 9 9 8 8 7 24/30 Read block 415 len 36864 25/30 Write block 260 len 40960 data: 10 10 10 11 10 9 8 9 8 10 26/30 Write block 77 len 16384 data: 6 4 5 4 27/30 Write block 431 len 12288 data: 7 8 8 28/30 Read block 591 len 12288 29/30 Read block 67 len 12288 30/30 Read block 16 len 16384 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 12 13 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:12.526443462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 103 103 103 175 175 175 F F F 3 060-079 104 104 104 177 177 177 T T T 4 080-099 104 104 104 177 177 177 T T T 5 100-119 104 104 104 178 178 178 F F F 6 120-139 100 100 100 168 168 168 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 104 104 104 177 177 177 T T T 11 220-239 92 92 92 155 155 155 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 104 104 104 177 177 177 T T T 14 280-299 104 104 104 177 177 177 T T T 15 300-319 101 101 101 170 170 170 T T T 16 320-339 103 103 103 175 175 175 F F F 17 340-359 101 101 101 170 170 170 F F F 18 360-379 104 104 104 177 177 177 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 100 100 100 168 168 168 T T T 21 420-439 100 100 100 167 167 167 T T T 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 104 104 104 177 177 177 F F F 26 520-539 103 103 103 175 175 175 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 103 103 103 175 175 175 F F F Max gen: 104, Max flush: 178 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 103 103 103 175 175 175 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 105 105 105 179 179 179 F F F 5 100-119 104 104 104 178 178 178 F F F 6 120-139 100 100 100 168 168 168 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 105 105 105 179 179 179 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 105 105 105 179 179 179 F F F 14 280-299 105 105 105 179 179 179 F F F 15 300-319 105 105 105 179 179 179 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 101 101 101 170 170 170 F F F 18 360-379 104 104 104 177 177 177 F F F 19 380-399 99 99 99 166 166 166 F F F 20 400-419 105 105 105 179 179 179 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 104 104 104 177 177 177 F F F 26 520-539 103 103 103 175 175 175 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 103 103 103 175 175 175 F F F Max gen: 105, Max flush: 179 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.813050738Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:16.856451044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.856826245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.888891928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.889460921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.889494619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.889508088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.889522067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} {"msg":"Crucible c7a0d708-e0a9-4e4f-9702-dfbd5eb0c9fa has session id: 2bf4d89b-8437-4155-ae40-2c425617c19c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.889998224Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Upstairs opts: Upstairs UUID: c7a0d708-e0a9-4e4f-9702-dfbd5eb0c9fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.890032432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Crucible stats registered with UUID: c7a0d708-e0a9-4e4f-9702-dfbd5eb0c9fa","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.890047512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920138712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920169771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} {"msg":"c7a0d708-e0a9-4e4f-9702-dfbd5eb0c9fa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920253326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920276315Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920303624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920325903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920343462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920360771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.92039106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920425118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920446497Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920463856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62338)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920492975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38669)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920526653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55452)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.920672616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.921298105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.921339433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.921369621Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.9214001Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.921428018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.921443268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.92200768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922045528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922063037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922537874Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922579792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.92261026Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922676177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922704786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922720425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922733165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922746824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[0]R flush_numbers[0..12]: [172, 166, 175, 179, 179, 178, 168, 170, 156, 173, 179, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922761033Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[0]R generation[0..12]: [102, 99, 103, 105, 105, 104, 100, 101, 93, 102, 105, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922775442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922790002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[1]R flush_numbers[0..12]: [172, 166, 175, 179, 179, 178, 168, 170, 156, 173, 179, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922803831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[1]R generation[0..12]: [102, 99, 103, 105, 105, 104, 100, 101, 93, 102, 105, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.92282145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922836279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[2]R flush_numbers[0..12]: [172, 166, 175, 179, 179, 178, 168, 170, 156, 173, 179, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922850139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[2]R generation[0..12]: [102, 99, 103, 105, 105, 104, 100, 101, 93, 102, 105, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922863858Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922877767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922890977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922904256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:08:16.922917145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922934935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"":"downstairs","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922948104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922960153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"0","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922973293Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"1","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922986362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"client":"2","":"downstairs","session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.922999621Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.923012021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"c7a0d708-e0a9-4e4f-9702-dfbd5eb0c9fa is now active with session: 2bf4d89b-8437-4155-ae40-2c425617c19c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.92302631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.923038639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606,"session_id":"2bf4d89b-8437-4155-ae40-2c425617c19c"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:16.923050919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 469 len 8192 data: 10 10 02/30 Read block 81 len 16384 03/30 Read block 163 len 12288 04/30 Write block 505 len 20480 data: 7 5 5 6 8 05/30 Write block 76 len 32768 data: 7 7 5 6 5 5 5 6 06/30 Read block 95 len 12288 07/30 Write block 299 len 16384 data: 10 10 10 9 08/30 Read block 445 len 8192 09/30 Write block 273 len 36864 data: 10 12 10 11 13 10 8 9 8 10/30 Read block 6 len 4096 11/30 Flush 12/30 Write block 108 len 12288 data: 11 14 14 13/30 Write block 456 len 32768 data: 6 7 6 7 9 8 7 9 14/30 Write block 262 len 16384 data: 11 12 11 10 15/30 Read block 157 len 28672 16/30 Write block 203 len 28672 data: 9 10 8 7 7 5 6 17/30 Read block 191 len 28672 18/30 Write block 349 len 16384 data: 11 9 10 10 19/30 Flush 20/30 Write block 118 len 40960 data: 10 9 8 5 4 5 6 6 7 10 21/30 Write block 41 len 20480 data: 12 14 15 17 13 22/30 Write block 593 len 20480 data: 8 7 7 5 6 23/30 Write block 215 len 12288 data: 9 11 8 24/30 Write block 122 len 8192 data: 5 6 25/30 Write block 348 len 32768 data: 10 12 10 11 11 13 10 8 26/30 Read block 105 len 20480 27/30 Write block 407 len 20480 data: 10 10 13 8 7 28/30 Write block 392 len 12288 data: 8 8 6 29/30 Read block 408 len 28672 30/30 Write block 167 len 40960 data: 8 10 10 11 10 11 11 11 10 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:16.941852334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 103 103 103 175 175 175 T T T 3 060-079 106 106 106 180 180 180 F F F 4 080-099 106 106 106 180 180 180 F F F 5 100-119 106 106 106 181 181 181 T T T 6 120-139 100 100 100 168 168 168 T T T 7 140-159 101 101 101 170 170 170 F F F 8 160-179 93 93 93 156 156 156 T T T 9 180-199 102 102 102 173 173 173 F F F 10 200-219 106 106 106 181 181 181 T T T 11 220-239 92 92 92 155 155 155 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 106 106 106 181 181 181 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 106 106 106 180 180 180 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 106 106 106 181 181 181 T T T 18 360-379 104 104 104 177 177 177 F F F 19 380-399 99 99 99 166 166 166 T T T 20 400-419 105 105 105 179 179 179 T T T 21 420-439 105 105 105 179 179 179 F F F 22 440-459 106 106 106 181 181 181 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 103 103 103 175 175 175 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 103 103 103 175 175 175 T T T Max gen: 106, Max flush: 181 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 172 172 172 F F F 1 020-039 99 99 99 166 166 166 F F F 2 040-059 107 107 107 182 182 182 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 106 106 106 180 180 180 F F F 5 100-119 107 107 107 182 182 182 F F F 6 120-139 107 107 107 182 182 182 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 107 107 107 182 182 182 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 107 107 107 182 182 182 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 104 104 104 176 176 176 F F F 13 260-279 106 106 106 181 181 181 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 106 106 106 180 180 180 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 107 107 107 182 182 182 F F F 18 360-379 104 104 104 177 177 177 F F F 19 380-399 107 107 107 182 182 182 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 106 106 106 181 181 181 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 103 103 103 175 175 175 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 97 97 97 164 164 164 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 107, Max flush: 182 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.270044596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.315688232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316387207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316424366Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316438605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316452714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} {"msg":"Crucible 2f40354a-0bc6-4ae4-bf78-cecf8eb2d9be has session id: 0b9879cc-de11-43bb-84f7-0c4d50afbed4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316906602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Upstairs opts: Upstairs UUID: 2f40354a-0bc6-4ae4-bf78-cecf8eb2d9be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.31694782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Crucible stats registered with UUID: 2f40354a-0bc6-4ae4-bf78-cecf8eb2d9be","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.316964259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347098727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347139205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} {"msg":"2f40354a-0bc6-4ae4-bf78-cecf8eb2d9be active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347168854Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347191693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347236631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347271439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347307507Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347346835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347384233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347419042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.3474522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347483368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41862)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347620412Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50235)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34766043Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40433)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.347682158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348226102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34825549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348271329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348296068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348447471Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348491169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.348954536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349055201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349088139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34947848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349504259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349518578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349592935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349612194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349626933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349639372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349652882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[0]R flush_numbers[0..12]: [172, 166, 182, 180, 180, 182, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349666691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[0]R generation[0..12]: [102, 99, 107, 106, 106, 107, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34968047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34969473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[1]R flush_numbers[0..12]: [172, 166, 182, 180, 180, 182, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349708549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[1]R generation[0..12]: [102, 99, 107, 106, 106, 107, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349722348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349739467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[2]R flush_numbers[0..12]: [172, 166, 182, 180, 180, 182, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349753617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[2]R generation[0..12]: [102, 99, 107, 106, 106, 107, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349767356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349781635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349794765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349808114Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349829173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349842112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"":"downstairs","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349855682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349867661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"0","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34988144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"1","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.34989463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"client":"2","":"downstairs","session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349907919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349920279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"2f40354a-0bc6-4ae4-bf78-cecf8eb2d9be is now active with session: 0b9879cc-de11-43bb-84f7-0c4d50afbed4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349932948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349948147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614,"session_id":"0b9879cc-de11-43bb-84f7-0c4d50afbed4"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:21.349960697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 40960 data: 14 11 9 6 5 8 9 8 7 9 02/30 Read block 33 len 16384 03/30 Write block 0 len 12288 data: 3 4 6 04/30 Flush 05/30 Flush 06/30 Write block 502 len 20480 data: 9 8 7 8 6 07/30 Read block 188 len 28672 08/30 Write block 98 len 8192 data: 7 6 09/30 Read block 469 len 20480 10/30 Read block 266 len 8192 11/30 Flush 12/30 Write block 82 len 16384 data: 6 7 7 5 13/30 Read block 489 len 12288 14/30 Write block 562 len 36864 data: 9 11 9 10 11 10 10 11 11 15/30 Write block 106 len 28672 data: 11 11 12 15 15 15 15 16/30 Write block 240 len 16384 data: 7 10 11 11 17/30 Write block 565 len 20480 data: 11 12 11 11 12 18/30 Write block 18 len 40960 data: 9 9 10 11 10 11 11 7 7 6 19/30 Read block 140 len 32768 20/30 Read block 271 len 32768 21/30 Write block 7 len 20480 data: 10 9 11 11 11 22/30 Read block 484 len 28672 23/30 Read block 73 len 36864 24/30 Read block 489 len 20480 25/30 Write block 388 len 20480 data: 6 6 7 7 9 26/30 Read block 555 len 36864 27/30 Write block 303 len 24576 data: 9 8 5 11 13 13 28/30 Read block 281 len 4096 29/30 Read block 307 len 36864 30/30 Write block 516 len 36864 data: 9 8 10 11 11 8 7 7 8 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:21.369746794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 183 183 183 T T T 1 020-039 99 99 99 166 166 166 T T T 2 040-059 107 107 107 182 182 182 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 108 108 108 185 185 185 T T T 5 100-119 107 107 107 182 182 182 T T T 6 120-139 107 107 107 182 182 182 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 107 107 107 182 182 182 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 107 107 107 182 182 182 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 104 104 104 176 176 176 T T T 13 260-279 106 106 106 181 181 181 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 106 106 106 180 180 180 T T T 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 107 107 107 182 182 182 T T T 20 400-419 107 107 107 182 182 182 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 106 106 106 181 181 181 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 108 108 108 185 185 185 T T T 26 520-539 103 103 103 175 175 175 T T T 27 540-559 104 104 104 176 176 176 F F F 28 560-579 97 97 97 164 164 164 T T T 29 580-599 107 107 107 182 182 182 F F F Max gen: 108, Max flush: 185 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 186 186 186 F F F 1 020-039 109 109 109 186 186 186 F F F 2 040-059 107 107 107 182 182 182 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 109 109 109 186 186 186 F F F 5 100-119 109 109 109 186 186 186 F F F 6 120-139 107 107 107 182 182 182 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 107 107 107 182 182 182 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 107 107 107 182 182 182 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 109 109 109 186 186 186 F F F 13 260-279 106 106 106 181 181 181 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 109 109 109 186 186 186 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 106 106 106 181 181 181 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 109 109 109 186 186 186 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 109 109 109 186 186 186 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 109, Max flush: 186 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.678245724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.724575146Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725377956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725421534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725436843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725452453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} {"msg":"Crucible 5754fb24-474f-48df-9b86-4c2623527256 has session id: 43d48a80-2495-4d02-8704-b3c337c34d4c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725887751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Upstairs opts: Upstairs UUID: 5754fb24-474f-48df-9b86-4c2623527256, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725935209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Crucible stats registered with UUID: 5754fb24-474f-48df-9b86-4c2623527256","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.725960948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757143564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757170643Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} {"msg":"5754fb24-474f-48df-9b86-4c2623527256 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75721938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757248299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757274998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757330745Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757370593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757398982Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75742936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757459069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75764717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757681848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63548)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757706487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34816)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757729426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60223)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.757754354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758192793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758263739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758305467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758328876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758343805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758357505Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75886453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758906957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.758952045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759401053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759441511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75947188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759533897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759564005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759579255Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759591894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759605593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[0]R flush_numbers[0..12]: [186, 186, 182, 180, 186, 186, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759619913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[0]R generation[0..12]: [109, 109, 107, 106, 109, 109, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759634002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759648181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[1]R flush_numbers[0..12]: [186, 186, 182, 180, 186, 186, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.7596619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[1]R generation[0..12]: [109, 109, 107, 106, 109, 109, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.75967585Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759690079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[2]R flush_numbers[0..12]: [186, 186, 182, 180, 186, 186, 182, 170, 182, 173, 182, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759707108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[2]R generation[0..12]: [109, 109, 107, 106, 109, 109, 107, 101, 107, 102, 107, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759721297Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759735697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759748726Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759827512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759857931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759912988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"":"downstairs","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759941247Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759966265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"0","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.759985304Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"1","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760002914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"client":"2","":"downstairs","session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760041182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760066151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"5754fb24-474f-48df-9b86-4c2623527256 is now active with session: 43d48a80-2495-4d02-8704-b3c337c34d4c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760089059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760102219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622,"session_id":"43d48a80-2495-4d02-8704-b3c337c34d4c"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:25.760122408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 211 len 24576 data: 9 8 10 8 10 12 02/30 Read block 533 len 36864 03/30 Write block 135 len 32768 data: 10 11 11 14 12 10 13 14 04/30 Write block 386 len 36864 data: 9 8 7 7 8 8 10 9 7 05/30 Read block 297 len 4096 06/30 Read block 294 len 4096 07/30 Flush 08/30 Write block 566 len 32768 data: 13 12 12 13 12 11 10 11 09/30 Write block 257 len 16384 data: 6 7 6 11 10/30 Write block 129 len 24576 data: 12 14 14 12 13 11 11/30 Write block 391 len 4096 data: 9 12/30 Read block 437 len 8192 13/30 Read block 395 len 20480 14/30 Read block 124 len 8192 15/30 Read block 6 len 8192 16/30 Read block 367 len 24576 17/30 Read block 476 len 36864 18/30 Flush 19/30 Write block 438 len 28672 data: 12 15 17 14 15 12 14 20/30 Read block 74 len 12288 21/30 Read block 550 len 4096 22/30 Write block 102 len 8192 data: 10 11 23/30 Read block 362 len 24576 24/30 Read block 36 len 12288 25/30 Read block 542 len 8192 26/30 Write block 41 len 16384 data: 13 15 16 18 27/30 Read block 187 len 4096 28/30 Read block 193 len 36864 29/30 Write block 436 len 40960 data: 9 11 13 16 18 15 16 13 15 13 30/30 Read block 8 len 36864 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:25.77959205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 186 186 186 F F F 1 020-039 109 109 109 186 186 186 F F F 2 040-059 107 107 107 182 182 182 T T T 3 060-079 106 106 106 180 180 180 F F F 4 080-099 109 109 109 186 186 186 F F F 5 100-119 109 109 109 186 186 186 T T T 6 120-139 110 110 110 188 188 188 F F F 7 140-159 110 110 110 187 187 187 F F F 8 160-179 107 107 107 182 182 182 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 110 110 110 188 188 188 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 110 110 110 188 188 188 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 105 105 105 179 179 179 T T T 22 440-459 106 106 106 181 181 181 T T T 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 109 109 109 186 186 186 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 110, Max flush: 188 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 186 186 186 F F F 1 020-039 109 109 109 186 186 186 F F F 2 040-059 111 111 111 189 189 189 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 109 109 109 186 186 186 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 110 110 110 187 187 187 F F F 8 160-179 107 107 107 182 182 182 F F F 9 180-199 102 102 102 173 173 173 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 92 92 92 155 155 155 F F F 12 240-259 110 110 110 188 188 188 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 110 110 110 188 188 188 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 111 111 111 189 189 189 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 109 109 109 186 186 186 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 111, Max flush: 189 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.051254771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.09622022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.096889257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.096928355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.096947354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.096962843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} {"msg":"Crucible 5741cad3-dad9-426d-8923-1735bcafbdaa has session id: bca5ae6f-5d04-46fd-ad76-bb0c381e0fec","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.097463859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Upstairs opts: Upstairs UUID: 5741cad3-dad9-426d-8923-1735bcafbdaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.097503457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Crucible stats registered with UUID: 5741cad3-dad9-426d-8923-1735bcafbdaa","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.097519526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.127866714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.127892392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} {"msg":"5741cad3-dad9-426d-8923-1735bcafbdaa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.127961989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.127990988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128015976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128031626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128045685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128059354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128072914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128086193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128100092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128117332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_connection connected from Ok(127.0.0.1:41851)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128394648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_connection connected from Ok(127.0.0.1:48768)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128437766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_connection connected from Ok(127.0.0.1:59713)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.128465834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129030816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129073574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129106813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129137101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.12916106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129175329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129657666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.129683944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.12977939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130246517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130274895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130292334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130357731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.13038731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130402589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130415308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130429248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[0]R flush_numbers[0..12]: [186, 186, 189, 180, 186, 189, 188, 187, 182, 173, 187, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130443417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[0]R generation[0..12]: [109, 109, 111, 106, 109, 111, 110, 110, 107, 102, 110, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130457436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130471626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[1]R flush_numbers[0..12]: [186, 186, 189, 180, 186, 189, 188, 187, 182, 173, 187, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130485585Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[1]R generation[0..12]: [109, 109, 111, 106, 109, 111, 110, 110, 107, 102, 110, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130499304Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130513234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[2]R flush_numbers[0..12]: [186, 186, 189, 180, 186, 189, 188, 187, 182, 173, 187, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130527003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[2]R generation[0..12]: [109, 109, 111, 106, 109, 111, 110, 110, 107, 102, 110, 92]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130540582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130558351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130572281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.13058577Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130598799Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130619728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"":"downstairs","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130632888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130644877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"0","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130658116Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"1","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130671306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"client":"2","":"downstairs","session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130684485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130696745Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"5741cad3-dad9-426d-8923-1735bcafbdaa is now active with session: bca5ae6f-5d04-46fd-ad76-bb0c381e0fec","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130709384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130721333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630,"session_id":"bca5ae6f-5d04-46fd-ad76-bb0c381e0fec"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:30.130733693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 32768 data: 6 7 8 7 8 8 9 11 02/30 Read block 331 len 20480 03/30 Read block 345 len 4096 04/30 Read block 30 len 8192 05/30 Read block 500 len 40960 06/30 Write block 488 len 28672 data: 7 7 8 9 9 10 6 07/30 Read block 597 len 8192 08/30 Write block 165 len 24576 data: 7 8 9 11 11 12 09/30 Write block 145 len 28672 data: 9 7 8 5 4 5 6 10/30 Read block 520 len 8192 11/30 Read block 141 len 4096 12/30 Read block 240 len 28672 13/30 Flush 14/30 Write block 249 len 40960 data: 8 9 8 8 8 4 5 6 7 8 15/30 Read block 305 len 16384 16/30 Read block 229 len 40960 17/30 Read block 287 len 28672 18/30 Write block 19 len 8192 data: 10 11 19/30 Read block 577 len 12288 20/30 Read block 312 len 40960 21/30 Read block 487 len 12288 22/30 Read block 238 len 24576 23/30 Flush 24/30 Write block 524 len 32768 data: 9 10 10 9 7 8 8 9 25/30 Write block 380 len 16384 data: 7 8 8 7 26/30 Read block 177 len 16384 27/30 Read block 581 len 24576 28/30 Write block 83 len 4096 data: 8 29/30 Read block 313 len 28672 30/30 Write block 226 len 28672 data: 9 7 7 8 9 9 10 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:30.151426335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 191 191 191 F F F 1 020-039 112 112 112 191 191 191 F F F 2 040-059 111 111 111 189 189 189 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 109 109 109 186 186 186 T T T 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 112 112 112 190 190 190 F F F 8 160-179 112 112 112 190 190 190 F F F 9 180-199 112 112 112 190 190 190 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 92 92 92 155 155 155 T T T 12 240-259 112 112 112 191 191 191 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 110 110 110 188 188 188 T T T 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 111 111 111 189 189 189 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 112 112 112 190 190 190 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 109 109 109 186 186 186 T T T 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 112, Max flush: 191 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 191 191 191 F F F 1 020-039 112 112 112 191 191 191 F F F 2 040-059 111 111 111 189 189 189 F F F 3 060-079 106 106 106 180 180 180 F F F 4 080-099 113 113 113 192 192 192 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 112 112 112 190 190 190 F F F 8 160-179 112 112 112 190 190 190 F F F 9 180-199 112 112 112 190 190 190 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 113 113 113 192 192 192 F F F 12 240-259 112 112 112 191 191 191 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 F F F 18 360-379 108 108 108 183 183 183 F F F 19 380-399 113 113 113 192 192 192 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 111 111 111 189 189 189 F F F 23 460-479 106 106 106 181 181 181 F F F 24 480-499 112 112 112 190 190 190 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 113, Max flush: 192 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.415188445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.459890207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.460528806Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.460566664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.460581553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.460595832Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} {"msg":"Crucible 0d46fef0-efac-4b8b-8f95-305bba617326 has session id: 70436318-026d-4f5a-827b-eff69d94db23","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.461107527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Upstairs opts: Upstairs UUID: 0d46fef0-efac-4b8b-8f95-305bba617326, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.461156205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Crucible stats registered with UUID: 0d46fef0-efac-4b8b-8f95-305bba617326","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.461186283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491228266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491263104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} {"msg":"0d46fef0-efac-4b8b-8f95-305bba617326 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491285623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491301863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491329241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.49134736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.49136485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491381819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491398638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491425987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491485024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491518002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_connection connected from Ok(127.0.0.1:45647)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.49155309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_connection connected from Ok(127.0.0.1:45182)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491587429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_connection connected from Ok(127.0.0.1:37372)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.491621537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492424578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492454956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492471045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492485035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492546112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.492588379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493109704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493151432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.49318528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493667626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493707914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493737913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493760652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493784101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493811079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493834338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493858687Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 189, 180, 192, 189, 188, 190, 190, 190, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493882376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[0]R generation[0..12]: [112, 112, 111, 106, 113, 111, 110, 112, 112, 112, 110, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493906785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493933493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 189, 180, 192, 189, 188, 190, 190, 190, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493960912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[1]R generation[0..12]: [112, 112, 111, 106, 113, 111, 110, 112, 112, 112, 110, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.493986571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494014429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 189, 180, 192, 189, 188, 190, 190, 190, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494042688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[2]R generation[0..12]: [112, 112, 111, 106, 113, 111, 110, 112, 112, 112, 110, 113]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:08:34.494072697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494106325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494135683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494163562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494195171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494224419Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"":"downstairs","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494253197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494279526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"0","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494313985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"1","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494332684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"client":"2","":"downstairs","session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494346893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494359572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"0d46fef0-efac-4b8b-8f95-305bba617326 is now active with session: 70436318-026d-4f5a-827b-eff69d94db23","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494372672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.494384851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638,"session_id":"70436318-026d-4f5a-827b-eff69d94db23"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:34.49439717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 476 len 36864 data: 5 6 7 7 9 12 11 9 8 02/30 Write block 164 len 40960 data: 8 8 9 10 12 12 13 11 12 12 03/30 Read block 148 len 36864 04/30 Read block 400 len 12288 05/30 Read block 216 len 36864 06/30 Write block 5 len 4096 data: 9 07/30 Read block 223 len 8192 08/30 Read block 109 len 8192 09/30 Write block 18 len 16384 data: 10 11 12 12 10/30 Write block 65 len 16384 data: 8 6 6 8 11/30 Read block 336 len 36864 12/30 Write block 34 len 36864 data: 8 12 12 13 12 10 11 14 16 13/30 Read block 70 len 24576 14/30 Read block 315 len 4096 15/30 Flush 16/30 Write block 254 len 16384 data: 5 6 7 8 17/30 Write block 444 len 32768 data: 16 14 14 10 9 4 3 2 18/30 Read block 139 len 40960 19/30 Flush 20/30 Flush 21/30 Write block 347 len 8192 data: 7 11 22/30 Read block 322 len 36864 23/30 Read block 494 len 36864 24/30 Read block 578 len 4096 25/30 Write block 370 len 4096 data: 10 26/30 Read block 201 len 40960 27/30 Write block 350 len 8192 data: 11 12 28/30 Write block 235 len 32768 data: 11 11 11 9 8 8 11 12 29/30 Read block 135 len 20480 30/30 Read block 515 len 16384 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:34.513625205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 114 114 114 193 193 193 F F F 2 040-059 114 114 114 193 193 193 F F F 3 060-079 114 114 114 193 193 193 F F F 4 080-099 113 113 113 192 192 192 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 112 112 112 190 190 190 F F F 8 160-179 114 114 114 193 193 193 F F F 9 180-199 112 112 112 190 190 190 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 113 113 113 192 192 192 T T T 12 240-259 114 114 114 194 194 194 T T T 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 108 108 108 183 183 183 T T T 18 360-379 108 108 108 183 183 183 T T T 19 380-399 113 113 113 192 192 192 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 114 114 114 193 193 193 F F F 24 480-499 114 114 114 193 193 193 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 114, Max flush: 194 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 114 114 114 193 193 193 F F F 2 040-059 114 114 114 193 193 193 F F F 3 060-079 114 114 114 193 193 193 F F F 4 080-099 113 113 113 192 192 192 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 112 112 112 190 190 190 F F F 8 160-179 114 114 114 193 193 193 F F F 9 180-199 112 112 112 190 190 190 F F F 10 200-219 110 110 110 187 187 187 F F F 11 220-239 115 115 115 195 195 195 F F F 12 240-259 115 115 115 195 195 195 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 115 115 115 195 195 195 F F F 18 360-379 115 115 115 195 195 195 F F F 19 380-399 113 113 113 192 192 192 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 114 114 114 193 193 193 F F F 24 480-499 114 114 114 193 193 193 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 115, Max flush: 195 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.778584306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.823243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.82386104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.823898458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.823913567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.823928227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} {"msg":"Crucible fc57c4a7-516a-4602-ae45-27210237356c has session id: 07c906ca-c686-4c2e-8829-7ed53633596e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.824452311Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Upstairs opts: Upstairs UUID: fc57c4a7-516a-4602-ae45-27210237356c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.824498179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Crucible stats registered with UUID: fc57c4a7-516a-4602-ae45-27210237356c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.824523817Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.854927122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.854957861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} {"msg":"fc57c4a7-516a-4602-ae45-27210237356c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855025238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855044747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855067735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855082415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855096414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855119613Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855134862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855149411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855163631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.85517764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34478)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855308174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46228)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855351402Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56398)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.855462556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856087196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856116244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856133963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856148473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856162242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856175801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856726024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856864387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.856889986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857253268Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857281977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857296526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857381302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857401431Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.85741525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857470477Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857490507Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[0]R flush_numbers[0..12]: [193, 193, 193, 193, 192, 189, 188, 190, 193, 190, 187, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857508135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[0]R generation[0..12]: [114, 114, 114, 114, 113, 111, 110, 112, 114, 112, 110, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857522575Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857536774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[1]R flush_numbers[0..12]: [193, 193, 193, 193, 192, 189, 188, 190, 193, 190, 187, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857550773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[1]R generation[0..12]: [114, 114, 114, 114, 113, 111, 110, 112, 114, 112, 110, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857564453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857578512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[2]R flush_numbers[0..12]: [193, 193, 193, 193, 192, 189, 188, 190, 193, 190, 187, 195]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857592391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[2]R generation[0..12]: [114, 114, 114, 114, 113, 111, 110, 112, 114, 112, 110, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857606211Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.85762018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.85763326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857646589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857659608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"}Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857680717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"":"downstairs","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857693926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857707826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"0","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857721205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"1","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857734375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"client":"2","":"downstairs","session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857747514Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857759853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"fc57c4a7-516a-4602-ae45-27210237356c is now active with session: 07c906ca-c686-4c2e-8829-7ed53633596e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857772742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857784842Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646,"session_id":"07c906ca-c686-4c2e-8829-7ed53633596e"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:38.857797111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 170 len 24576 data: 14 12 13 13 12 11 02/30 Write block 147 len 16384 data: 9 6 5 6 03/30 Read block 481 len 24576 04/30 Read block 279 len 40960 05/30 Write block 181 len 4096 data: 8 06/30 Read block 456 len 40960 07/30 Read block 180 len 36864 08/30 Read block 334 len 28672 09/30 Read block 516 len 28672 10/30 Read block 423 len 8192 11/30 Read block 394 len 32768 12/30 Write block 353 len 24576 data: 15 12 10 7 6 9 13/30 Write block 218 len 40960 data: 10 10 7 7 5 4 5 6 10 8 14/30 Read block 195 len 40960 15/30 Write block 378 len 24576 data: 10 8 8 9 9 8 16/30 Write block 239 len 20480 data: 9 9 12 13 12 17/30 Write block 473 len 8192 data: 7 7 18/30 Read block 93 len 8192 19/30 Write block 186 len 4096 data: 8 20/30 Flush 21/30 Write block 473 len 8192 data: 8 8 22/30 Read block 507 len 24576 23/30 Read block 196 len 40960 24/30 Read block 275 len 12288 25/30 Write block 57 len 4096 data: 7 26/30 Write block 44 len 24576 data: 19 14 10 10 9 8 27/30 Write block 521 len 32768 data: 9 8 8 10 11 11 10 8 28/30 Read block 107 len 24576 29/30 Read block 471 len 36864 30/30 Write block 356 len 12288 data: 8 7 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:38.876751519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 114 114 114 193 193 193 F F F 2 040-059 114 114 114 193 193 193 T T T 3 060-079 114 114 114 193 193 193 F F F 4 080-099 113 113 113 192 192 192 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 116 116 116 196 196 196 F F F 8 160-179 116 116 116 196 196 196 F F F 9 180-199 116 116 116 196 196 196 F F F 10 200-219 116 116 116 196 196 196 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 116 116 116 196 196 196 T T T 18 360-379 116 116 116 196 196 196 F F F 19 380-399 116 116 116 196 196 196 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 116 116 116 196 196 196 T T T 24 480-499 114 114 114 193 193 193 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 113 113 113 192 192 192 T T T 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 116, Max flush: 196 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 114 114 114 193 193 193 F F F 2 040-059 117 117 117 197 197 197 F F F 3 060-079 114 114 114 193 193 193 F F F 4 080-099 113 113 113 192 192 192 F F F 5 100-119 111 111 111 189 189 189 F F F 6 120-139 110 110 110 188 188 188 F F F 7 140-159 116 116 116 196 196 196 F F F 8 160-179 116 116 116 196 196 196 F F F 9 180-199 116 116 116 196 196 196 F F F 10 200-219 116 116 116 196 196 196 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 110 110 110 188 188 188 F F F 14 280-299 106 106 106 180 180 180 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 F F F 17 340-359 117 117 117 197 197 197 F F F 18 360-379 116 116 116 196 196 196 F F F 19 380-399 116 116 116 196 196 196 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 117 117 117 197 197 197 F F F 24 480-499 114 114 114 193 193 193 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 117 117 117 197 197 197 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 117, Max flush: 197 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.139117928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:43.183213479Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.18361337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.216199018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.216802168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.216838626Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.216853005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.216867735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} {"msg":"Crucible 26c4497f-a071-4752-82b9-a30539df8772 has session id: 55cadf67-2937-4e8c-a5ec-6369642e8b45","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.217249396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Upstairs opts: Upstairs UUID: 26c4497f-a071-4752-82b9-a30539df8772, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.217296174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Crucible stats registered with UUID: 26c4497f-a071-4752-82b9-a30539df8772","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.217321802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247761706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247793384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} {"msg":"26c4497f-a071-4752-82b9-a30539df8772 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247831872Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247849481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.24787735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247894919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247911838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247941117Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.247975055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248004974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248038882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.24807285Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:38151)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248106639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:58901)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248139337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:48480)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248336328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248797905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248869261Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248908209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248931188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248957717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.248973466Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.24951303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.249567987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.249590376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250097501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.25012538Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250140779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250208745Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250228204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250242174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250254513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250267962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[0]R flush_numbers[0..12]: [193, 193, 197, 193, 192, 189, 188, 196, 196, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250281842Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[0]R generation[0..12]: [114, 114, 117, 114, 113, 111, 110, 116, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250298751Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.25031332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[1]R flush_numbers[0..12]: [193, 193, 197, 193, 192, 189, 188, 196, 196, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250327319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[1]R generation[0..12]: [114, 114, 117, 114, 113, 111, 110, 116, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250340999Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250354818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[2]R flush_numbers[0..12]: [193, 193, 197, 193, 192, 189, 188, 196, 196, 196, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250368428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[2]R generation[0..12]: [114, 114, 117, 114, 113, 111, 110, 116, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250387977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250414335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250440634Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:08:43.250466633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250497671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.25051917Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"":"downstairs","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.25053322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250551188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"0","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250579807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"1","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250607426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"client":"2","":"downstairs","session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250626875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250642244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"26c4497f-a071-4752-82b9-a30539df8772 is now active with session: 55cadf67-2937-4e8c-a5ec-6369642e8b45","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250666193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.250690342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654,"session_id":"55cadf67-2937-4e8c-a5ec-6369642e8b45"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:43.25071484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 273 len 40960 data: 11 13 11 12 14 11 9 10 9 7 02/30 Read block 141 len 40960 03/30 Read block 92 len 24576 04/30 Write block 460 len 32768 data: 10 9 8 10 8 10 11 10 05/30 Flush 06/30 Write block 108 len 32768 data: 13 16 16 16 16 11 10 10 07/30 Read block 468 len 36864 08/30 Read block 66 len 8192 09/30 Read block 214 len 20480 10/30 Read block 220 len 36864 11/30 Read block 33 len 28672 12/30 Write block 131 len 24576 data: 15 13 14 12 11 12 13/30 Write block 85 len 12288 data: 6 4 6 14/30 Write block 492 len 12288 data: 10 11 7 15/30 Read block 354 len 16384 16/30 Write block 32 len 12288 data: 7 6 9 17/30 Read block 354 len 8192 18/30 Write block 94 len 16384 data: 7 9 10 7 19/30 Write block 68 len 24576 data: 9 10 10 8 8 9 20/30 Read block 379 len 40960 21/30 Flush 22/30 Write block 337 len 16384 data: 6 8 8 7 23/30 Read block 215 len 8192 24/30 Read block 380 len 16384 25/30 Read block 96 len 40960 26/30 Write block 52 len 20480 data: 10 9 11 7 8 27/30 Read block 336 len 32768 28/30 Write block 197 len 20480 data: 10 9 9 9 10 29/30 Write block 68 len 4096 data: 10 30/30 Read block 531 len 36864 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 9 9 6 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:43.270898378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 118 118 118 199 199 199 F F F 2 040-059 117 117 117 197 197 197 T T T 3 060-079 118 118 118 199 199 199 T T T 4 080-099 118 118 118 199 199 199 F F F 5 100-119 118 118 118 199 199 199 F F F 6 120-139 118 118 118 199 199 199 F F F 7 140-159 116 116 116 196 196 196 F F F 8 160-179 116 116 116 196 196 196 F F F 9 180-199 116 116 116 196 196 196 T T T 10 200-219 116 116 116 196 196 196 T T T 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 118 118 118 198 198 198 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 103 103 103 175 175 175 T T T 17 340-359 117 117 117 197 197 197 T T T 18 360-379 116 116 116 196 196 196 F F F 19 380-399 116 116 116 196 196 196 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 118 118 118 199 199 199 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 117 117 117 197 197 197 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 118, Max flush: 199 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 193 193 193 F F F 1 020-039 118 118 118 199 199 199 F F F 2 040-059 119 119 119 200 200 200 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 118 118 118 199 199 199 F F F 5 100-119 118 118 118 199 199 199 F F F 6 120-139 118 118 118 199 199 199 F F F 7 140-159 116 116 116 196 196 196 F F F 8 160-179 116 116 116 196 196 196 F F F 9 180-199 119 119 119 200 200 200 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 118 118 118 198 198 198 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 119 119 119 200 200 200 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 116 116 116 196 196 196 F F F 19 380-399 116 116 116 196 196 196 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 118 118 118 199 199 199 F F F 25 500-519 109 109 109 186 186 186 F F F 26 520-539 117 117 117 197 197 197 F F F 27 540-559 104 104 104 176 176 176 F F F 28 560-579 110 110 110 188 188 188 F F F 29 580-599 107 107 107 182 182 182 F F F Max gen: 119, Max flush: 200 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.584295127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.629854367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.630584191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.630625399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.630640788Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.630655488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} {"msg":"Crucible fb7051b4-321a-4081-b6e6-2d6efa9bcda2 has session id: 9908f040-a2be-4403-8bc5-0f3d04548644","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.631090436Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Upstairs opts: Upstairs UUID: fb7051b4-321a-4081-b6e6-2d6efa9bcda2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.631134024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Crucible stats registered with UUID: fb7051b4-321a-4081-b6e6-2d6efa9bcda2","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.631151563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661394526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661429035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} {"msg":"fb7051b4-321a-4081-b6e6-2d6efa9bcda2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661452723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661469382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661497131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66151945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661537319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661554428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661571397Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661588017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661615645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661649044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_connection connected from Ok(127.0.0.1:45870)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661753779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_connection connected from Ok(127.0.0.1:60530)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661800766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_connection connected from Ok(127.0.0.1:33581)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.661947579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.662484153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.662524841Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66254022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.662554189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.662709492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66274552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663218276Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663249055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663265354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663792208Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663821537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663836876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663906813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663926462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663940401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66395272Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66396628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[0]R flush_numbers[0..12]: [193, 199, 200, 200, 199, 199, 199, 196, 196, 200, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.663992329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[0]R generation[0..12]: [114, 118, 119, 119, 118, 118, 118, 116, 116, 119, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664022877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664038926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[1]R flush_numbers[0..12]: [193, 199, 200, 200, 199, 199, 199, 196, 196, 200, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664052986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[1]R generation[0..12]: [114, 118, 119, 119, 118, 118, 118, 116, 116, 119, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664070815Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664085384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[2]R flush_numbers[0..12]: [193, 199, 200, 200, 199, 199, 199, 196, 196, 200, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664099253Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[2]R generation[0..12]: [114, 118, 119, 119, 118, 118, 118, 116, 116, 119, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664113052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664127112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664140181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664153581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.66417452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664187619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"":"downstairs","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664201098Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664213288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"0","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664226707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"1","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664239986Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"client":"2","":"downstairs","session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664253396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664267565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"fb7051b4-321a-4081-b6e6-2d6efa9bcda2 is now active with session: 9908f040-a2be-4403-8bc5-0f3d04548644","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664280804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664292824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662,"session_id":"9908f040-a2be-4403-8bc5-0f3d04548644"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:47.664305083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 133 len 36864 data: 15 13 12 13 12 15 13 11 14 02/30 Write block 595 len 8192 data: 8 6 03/30 Read block 197 len 8192 04/30 Write block 515 len 40960 data: 9 10 9 11 12 12 10 9 9 11 05/30 Read block 18 len 36864 06/30 Write block 470 len 8192 data: 11 10 07/30 Read block 356 len 28672 08/30 Read block 116 len 4096 09/30 Read block 113 len 8192 10/30 Read block 178 len 40960 11/30 Flush 12/30 Write block 159 len 24576 data: 10 8 9 8 5 9 13/30 Write block 587 len 40960 data: 9 9 10 11 10 9 9 8 9 7 14/30 Read block 172 len 32768 15/30 Read block 259 len 8192 16/30 Write block 554 len 36864 data: 4 4 5 7 11 10 10 10 10 17/30 Read block 118 len 4096 18/30 Read block 260 len 40960 19/30 Write block 193 len 4096 data: 8 20/30 Read block 215 len 40960 21/30 Write block 96 len 4096 data: 11 22/30 Read block 597 len 12288 23/30 Write block 392 len 20480 data: 11 10 8 4 5 24/30 Read block 383 len 32768 25/30 Write block 18 len 12288 data: 11 12 13 26/30 Flush 27/30 Write block 538 len 8192 data: 6 8 28/30 Read block 590 len 12288 29/30 Read block 86 len 40960 30/30 Write block 385 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 10 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 3 2 2 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:47.683576235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 120 120 120 202 202 202 F F F 2 040-059 119 119 119 200 200 200 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 120 120 120 202 202 202 F F F 5 100-119 118 118 118 199 199 199 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 120 120 120 202 202 202 F F F 9 180-199 120 120 120 202 202 202 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 118 118 118 198 198 198 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 119 119 119 200 200 200 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 116 116 116 196 196 196 F F F 19 380-399 120 120 120 202 202 202 T T T 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 120 120 120 201 201 201 F F F 24 480-499 118 118 118 199 199 199 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 120 120 120 201 201 201 T T T 27 540-559 120 120 120 202 202 202 F F F 28 560-579 120 120 120 202 202 202 F F F 29 580-599 120 120 120 202 202 202 F F F Max gen: 120, Max flush: 202 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 120 120 120 202 202 202 F F F 2 040-059 119 119 119 200 200 200 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 120 120 120 202 202 202 F F F 5 100-119 118 118 118 199 199 199 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 120 120 120 202 202 202 F F F 9 180-199 120 120 120 202 202 202 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 118 118 118 198 198 198 F F F 15 300-319 109 109 109 186 186 186 F F F 16 320-339 119 119 119 200 200 200 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 116 116 116 196 196 196 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 114 114 114 194 194 194 F F F 23 460-479 120 120 120 201 201 201 F F F 24 480-499 118 118 118 199 199 199 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 121 121 121 203 203 203 F F F 27 540-559 120 120 120 202 202 202 F F F 28 560-579 120 120 120 202 202 202 F F F 29 580-599 120 120 120 202 202 202 F F F Max gen: 121, Max flush: 203 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.94010026Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.992060176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.992660046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.992698584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.992728683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.992757371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} {"msg":"Crucible d3fc5aa1-c363-4b57-9903-67f846a55554 has session id: dc7361e4-1ed7-487e-a237-e2a1904d82c3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.993149862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Upstairs opts: Upstairs UUID: d3fc5aa1-c363-4b57-9903-67f846a55554, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.99319553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Crucible stats registered with UUID: d3fc5aa1-c363-4b57-9903-67f846a55554","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:51.993222988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024689731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024722599Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} {"msg":"d3fc5aa1-c363-4b57-9903-67f846a55554 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024753328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024770457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024798196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024816105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024833614Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024850673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024867722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024884522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02491837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024949388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40289)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.024979197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60053)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025068442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53493)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025224735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025789407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025830645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025908991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025951279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025967358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.025981348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.026508862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02655022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.026580378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02715956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027193638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027209427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027222167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027235976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027249425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027261605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027274794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[0]R flush_numbers[0..12]: [202, 202, 200, 200, 202, 199, 201, 202, 202, 202, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027288913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[0]R generation[0..12]: [120, 120, 119, 119, 120, 118, 120, 120, 120, 120, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027303023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027317132Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[1]R flush_numbers[0..12]: [202, 202, 200, 200, 202, 199, 201, 202, 202, 202, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027330971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[1]R generation[0..12]: [120, 120, 119, 119, 120, 118, 120, 120, 120, 120, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027344681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02735863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[2]R flush_numbers[0..12]: [202, 202, 200, 200, 202, 199, 201, 202, 202, 202, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027372399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[2]R generation[0..12]: [120, 120, 119, 119, 120, 118, 120, 120, 120, 120, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027388148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027402608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027415627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027429166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Next flush: 204"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027451945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027465455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"":"downstairs","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027478804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027490903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"0","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027504023Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"1","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027517382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"client":"2","":"downstairs","session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027530892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027543401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"d3fc5aa1-c363-4b57-9903-67f846a55554 is now active with session: dc7361e4-1ed7-487e-a237-e2a1904d82c3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02755612Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.02756835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670,"session_id":"dc7361e4-1ed7-487e-a237-e2a1904d82c3"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:52.027580919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 482 len 32768 data: 12 10 9 7 8 7 8 8 02/30 Read block 108 len 8192 03/30 Flush 04/30 Write block 458 len 28672 data: 7 8 11 10 9 11 9 05/30 Write block 280 len 16384 data: 11 10 8 8 06/30 Read block 294 len 12288 07/30 Write block 85 len 32768 data: 7 5 7 9 9 10 10 10 08/30 Write block 370 len 28672 data: 11 10 9 9 8 10 12 09/30 Write block 505 len 4096 data: 9 10/30 Read block 119 len 32768 11/30 Read block 238 len 16384 12/30 Read block 147 len 28672 13/30 Read block 178 len 20480 14/30 Write block 535 len 8192 data: 11 10 15/30 Write block 28 len 4096 data: 6 16/30 Flush 17/30 Write block 331 len 36864 data: 8 8 9 7 6 6 7 9 9 18/30 Flush 19/30 Write block 105 len 28672 data: 13 12 12 14 17 17 17 20/30 Read block 176 len 20480 21/30 Read block 204 len 20480 22/30 Write block 49 len 32768 data: 9 9 8 11 10 12 8 9 23/30 Write block 184 len 32768 data: 10 12 9 6 6 5 6 8 24/30 Read block 373 len 12288 25/30 Read block 446 len 24576 26/30 Write block 302 len 28672 data: 10 10 9 6 12 14 14 27/30 Write block 180 len 36864 data: 9 9 9 9 11 13 10 7 7 28/30 Write block 489 len 12288 data: 9 9 10 29/30 Read block 312 len 28672 30/30 Read block 473 len 8192 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:52.04728915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 122 122 122 205 205 205 F F F 2 040-059 119 119 119 200 200 200 T T T 3 060-079 119 119 119 200 200 200 F F F 4 080-099 122 122 122 205 205 205 F F F 5 100-119 118 118 118 199 199 199 T T T 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 120 120 120 202 202 202 F F F 9 180-199 120 120 120 202 202 202 T T T 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 109 109 109 186 186 186 T T T 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 122 122 122 205 205 205 F F F 23 460-479 122 122 122 205 205 205 F F F 24 480-499 122 122 122 204 204 204 T T T 25 500-519 122 122 122 205 205 205 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 120 120 120 202 202 202 F F F 28 560-579 120 120 120 202 202 202 F F F 29 580-599 120 120 120 202 202 202 F F F Max gen: 122, Max flush: 206 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 122 122 122 205 205 205 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 122 122 122 205 205 205 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 120 120 120 202 202 202 F F F 9 180-199 123 123 123 207 207 207 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 118 118 118 198 198 198 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 123 123 123 207 207 207 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 107 107 107 182 182 182 F F F 21 420-439 111 111 111 189 189 189 F F F 22 440-459 122 122 122 205 205 205 F F F 23 460-479 122 122 122 205 205 205 F F F 24 480-499 123 123 123 207 207 207 F F F 25 500-519 122 122 122 205 205 205 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 120 120 120 202 202 202 F F F 28 560-579 120 120 120 202 202 202 F F F 29 580-599 120 120 120 202 202 202 F F F Max gen: 123, Max flush: 207 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.317942061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:56.362197655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.362600475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.395672079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396282969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396319087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396336826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396351896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} {"msg":"Crucible 0224ec05-f9ec-4d32-ae8b-d71acb3943a9 has session id: 598d9817-6868-4efb-a0f9-78c36718638a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396785974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Upstairs opts: Upstairs UUID: 0224ec05-f9ec-4d32-ae8b-d71acb3943a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396835172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Crucible stats registered with UUID: 0224ec05-f9ec-4d32-ae8b-d71acb3943a9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.396854171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427204279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427248816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} {"msg":"0224ec05-f9ec-4d32-ae8b-d71acb3943a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427319143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427347772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.42737275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427390759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427410729Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427424628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427524953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427562111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427601829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427639337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55355)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427673705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36631)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.427703724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54552)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.42779173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428293465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428323144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428468926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428499225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428513574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428534033Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428950603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.428984591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429234159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429581362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.42962165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429640059Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429653338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429667167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429680447Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429692546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429705845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[0]R flush_numbers[0..12]: [202, 205, 207, 200, 205, 207, 201, 202, 202, 207, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429766782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[0]R generation[0..12]: [120, 122, 123, 119, 122, 123, 120, 120, 120, 123, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429787552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429802271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[1]R flush_numbers[0..12]: [202, 205, 207, 200, 205, 207, 201, 202, 202, 207, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.42981617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[1]R generation[0..12]: [120, 122, 123, 119, 122, 123, 120, 120, 120, 123, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.42983009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429844259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[2]R flush_numbers[0..12]: [202, 205, 207, 200, 205, 207, 201, 202, 202, 207, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429858148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[2]R generation[0..12]: [120, 122, 123, 119, 122, 123, 120, 120, 120, 123, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429874417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429888747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429915445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429929294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429942394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429955583Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"":"downstairs","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2024-11-26T22:08:56.429968753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.429992142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"0","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.430005871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"1","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.43001908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"client":"2","":"downstairs","session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.43003238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.430044849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"0224ec05-f9ec-4d32-ae8b-d71acb3943a9 is now active with session: 598d9817-6868-4efb-a0f9-78c36718638a","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.430057648Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.430069658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678,"session_id":"598d9817-6868-4efb-a0f9-78c36718638a"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:08:56.430082047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 270 len 20480 data: 9 7 8 12 14 02/30 Write block 310 len 24576 data: 12 13 13 13 13 12 03/30 Read block 230 len 28672 04/30 Read block 238 len 4096 05/30 Read block 217 len 12288 06/30 Read block 3 len 40960 07/30 Write block 518 len 28672 data: 12 13 13 11 10 10 12 08/30 Read block 401 len 28672 09/30 Read block 178 len 36864 10/30 Flush 11/30 Write block 410 len 16384 data: 9 8 8 7 12/30 Write block 463 len 16384 data: 12 10 11 12 13/30 Read block 323 len 4096 14/30 Write block 26 len 8192 data: 8 7 15/30 Write block 160 len 20480 data: 9 10 9 6 10 16/30 Read block 373 len 40960 17/30 Write block 539 len 12288 data: 9 7 8 18/30 Write block 439 len 40960 data: 17 19 16 17 14 17 15 15 11 10 19/30 Read block 444 len 32768 20/30 Flush 21/30 Write block 270 len 8192 data: 10 8 22/30 Read block 168 len 8192 23/30 Flush 24/30 Write block 89 len 40960 data: 10 11 11 11 8 8 10 12 8 8 25/30 Write block 568 len 4096 data: 13 26/30 Write block 33 len 16384 data: 7 10 13 13 27/30 Read block 118 len 24576 28/30 Read block 99 len 16384 29/30 Write block 193 len 8192 data: 9 9 30/30 Read block 494 len 36864 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:08:56.44833646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 124 124 124 209 209 209 T T T 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 122 122 122 205 205 205 T T T 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 124 124 124 209 209 209 F F F 9 180-199 123 123 123 207 207 207 T T T 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 124 124 124 208 208 208 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 124 124 124 209 209 209 F F F 22 440-459 124 124 124 209 209 209 F F F 23 460-479 124 124 124 209 209 209 F F F 24 480-499 123 123 123 207 207 207 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 124 124 124 209 209 209 F F F 27 540-559 124 124 124 209 209 209 F F F 28 560-579 120 120 120 202 202 202 T T T 29 580-599 120 120 120 202 202 202 F F F Max gen: 124, Max flush: 210 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 202 202 202 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 202 202 202 F F F 8 160-179 124 124 124 209 209 209 F F F 9 180-199 125 125 125 211 211 211 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 124 124 124 208 208 208 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 124 124 124 209 209 209 F F F 22 440-459 124 124 124 209 209 209 F F F 23 460-479 124 124 124 209 209 209 F F F 24 480-499 123 123 123 207 207 207 F F F 25 500-519 124 124 124 208 208 208 F F F 26 520-539 124 124 124 209 209 209 F F F 27 540-559 124 124 124 209 209 209 F F F 28 560-579 125 125 125 211 211 211 F F F 29 580-599 120 120 120 202 202 202 F F F Max gen: 125, Max flush: 211 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.743896646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:00.788064243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.788453484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.820725427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821309488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821345317Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821359836Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821374565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} {"msg":"Crucible 4731b865-2234-4f02-8280-3950b76e7986 has session id: 497a0dd9-b895-4058-bc5f-a29905d4c85f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821816323Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Upstairs opts: Upstairs UUID: 4731b865-2234-4f02-8280-3950b76e7986, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.821862481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Crucible stats registered with UUID: 4731b865-2234-4f02-8280-3950b76e7986","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.82188324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.851972381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852038088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} {"msg":"4731b865-2234-4f02-8280-3950b76e7986 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852060287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852089605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852137383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852176091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852212299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852248827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852285495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852321114Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852353722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852465716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64544)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852511774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60934)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852540993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56634)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.852563182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.85300931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853040248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853064197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853079516Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853100865Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853121584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853718915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853749003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.853764663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854328795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854359983Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854375962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854392582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854407151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.85442108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.85443376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854447489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[0]R flush_numbers[0..12]: [202, 211, 207, 200, 211, 207, 201, 202, 209, 211, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854461618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[0]R generation[0..12]: [120, 125, 123, 119, 125, 123, 120, 120, 124, 125, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854475598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854489847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[1]R flush_numbers[0..12]: [202, 211, 207, 200, 211, 207, 201, 202, 209, 211, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854503766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[1]R generation[0..12]: [120, 125, 123, 119, 125, 123, 120, 120, 124, 125, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854518295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854532435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[2]R flush_numbers[0..12]: [202, 211, 207, 200, 211, 207, 201, 202, 209, 211, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854546324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[2]R generation[0..12]: [120, 125, 123, 119, 125, 123, 120, 120, 124, 125, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854560293Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854574503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854589502Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairsWait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854611501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.85462469Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.85463785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"":"downstairs","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854651179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854663428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"0","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854676858Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"1","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854690077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"client":"2","":"downstairs","session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854703457Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854715826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"4731b865-2234-4f02-8280-3950b76e7986 is now active with session: 497a0dd9-b895-4058-bc5f-a29905d4c85f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854728765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854740895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686,"session_id":"497a0dd9-b895-4058-bc5f-a29905d4c85f"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:00.854753144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 555 len 8192 data: 5 6 02/30 Read block 423 len 4096 03/30 Flush 04/30 Write block 522 len 36864 data: 11 11 13 12 12 11 9 9 9 05/30 Read block 26 len 28672 06/30 Read block 490 len 8192 07/30 Read block 162 len 20480 08/30 Read block 509 len 16384 09/30 Read block 419 len 12288 10/30 Write block 581 len 8192 data: 10 10 11/30 Read block 386 len 16384 12/30 Write block 140 len 24576 data: 12 15 15 14 11 10 13/30 Write block 537 len 24576 data: 9 7 10 8 9 10 14/30 Write block 180 len 32768 data: 10 10 10 10 12 14 11 8 15/30 Write block 500 len 16384 data: 10 9 10 9 16/30 Write block 464 len 36864 data: 11 12 13 11 7 11 12 11 8 17/30 Read block 448 len 8192 18/30 Flush 19/30 Write block 497 len 16384 data: 11 11 10 11 20/30 Read block 431 len 24576 21/30 Write block 546 len 16384 data: 13 13 12 8 22/30 Flush 23/30 Write block 9 len 20480 data: 12 12 12 9 12 24/30 Read block 395 len 40960 25/30 Flush 26/30 Write block 253 len 8192 data: 9 6 27/30 Read block 161 len 8192 28/30 Write block 444 len 20480 data: 18 16 16 12 11 29/30 Write block 523 len 36864 data: 12 14 13 13 12 10 10 10 10 30/30 Read block 313 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:00.873229326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 124 124 124 209 209 209 F F F 9 180-199 126 126 126 213 213 213 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 116 116 116 196 196 196 T T T 13 260-279 124 124 124 210 210 210 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 124 124 124 208 208 208 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 124 124 124 209 209 209 F F F 22 440-459 124 124 124 209 209 209 T T T 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 214 214 214 F F F 25 500-519 126 126 126 214 214 214 F F F 26 520-539 126 126 126 213 213 213 T T T 27 540-559 126 126 126 214 214 214 F F F 28 560-579 125 125 125 211 211 211 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 126, Max flush: 215 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 124 124 124 209 209 209 F F F 9 180-199 126 126 126 213 213 213 F F F 10 200-219 119 119 119 200 200 200 F F F 11 220-239 116 116 116 196 196 196 F F F 12 240-259 127 127 127 216 216 216 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 122 122 122 205 205 205 F F F 15 300-319 124 124 124 208 208 208 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 124 124 124 209 209 209 F F F 22 440-459 127 127 127 216 216 216 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 214 214 214 F F F 25 500-519 126 126 126 214 214 214 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 125 125 125 211 211 211 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 127, Max flush: 216 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.161640453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.206264119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.206936426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.206972104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.206986124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.207000283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} {"msg":"Crucible 8c86d7d8-8b20-48e1-b831-f3745c1d76b3 has session id: a531bb82-a333-4c91-9183-33a1a7c7c225","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.207365025Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Upstairs opts: Upstairs UUID: 8c86d7d8-8b20-48e1-b831-f3745c1d76b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.207413233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Crucible stats registered with UUID: 8c86d7d8-8b20-48e1-b831-f3745c1d76b3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.207441571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237696354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237730602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} {"msg":"8c86d7d8-8b20-48e1-b831-f3745c1d76b3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237791199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237819238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237844196Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237859206Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237873375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237887164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237901154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237914523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237928362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.237941901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_connection connected from Ok(127.0.0.1:47798)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.238101234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_connection connected from Ok(127.0.0.1:48541)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.238330773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_connection connected from Ok(127.0.0.1:50703)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.23837323Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.238813159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.238841847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.238960371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239017189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239047467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239063576Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239411789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239548953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.239858097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240024289Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240063977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240095046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240153933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240185021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240200121Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240319295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240342843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[0]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 213, 209, 213, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240357823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[0]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 126, 124, 126, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240371952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240386371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[1]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 213, 209, 213, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240400321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[1]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 126, 124, 126, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.24041422Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240429329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[2]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 213, 209, 213, 200, 196]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240443258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[2]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 126, 124, 126, 119, 116]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240457098Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240471097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240484307Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240497586Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240510635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240523505Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"":"downstairs","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240538424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-11-26T22:09:05.240562103Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"0","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240575852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"1","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240589171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"client":"2","":"downstairs","session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240602461Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.24061475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"8c86d7d8-8b20-48e1-b831-f3745c1d76b3 is now active with session: a531bb82-a333-4c91-9183-33a1a7c7c225","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240627509Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240639609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694,"session_id":"a531bb82-a333-4c91-9183-33a1a7c7c225"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:05.240652058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 460 len 32768 data: 12 11 10 13 12 13 14 12 02/30 Read block 323 len 16384 03/30 Read block 270 len 16384 04/30 Read block 313 len 4096 05/30 Read block 523 len 4096 06/30 Read block 481 len 28672 07/30 Read block 266 len 4096 08/30 Write block 223 len 28672 data: 5 6 7 11 9 8 9 09/30 Read block 273 len 16384 10/30 Flush 11/30 Flush 12/30 Write block 431 len 24576 data: 8 9 9 8 10 10 13/30 Write block 214 len 32768 data: 9 11 13 9 11 11 8 8 14/30 Read block 507 len 12288 15/30 Read block 457 len 16384 16/30 Read block 28 len 28672 17/30 Write block 174 len 40960 data: 13 12 9 7 7 8 11 11 11 11 18/30 Flush 19/30 Write block 297 len 4096 data: 14 20/30 Write block 222 len 4096 data: 6 21/30 Read block 111 len 20480 22/30 Read block 298 len 20480 23/30 Read block 268 len 24576 24/30 Read block 414 len 32768 25/30 Read block 266 len 36864 26/30 Read block 231 len 36864 27/30 Read block 443 len 12288 28/30 Write block 301 len 8192 data: 11 11 29/30 Write block 154 len 12288 data: 6 8 8 30/30 Read block 426 len 40960 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:05.260664994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 126 126 126 213 213 213 T T T 8 160-179 128 128 128 219 219 219 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 128 128 128 219 219 219 F F F 11 220-239 128 128 128 219 219 219 T T T 12 240-259 127 127 127 216 216 216 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 122 122 122 205 205 205 T T T 15 300-319 124 124 124 208 208 208 T T T 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 128 128 128 219 219 219 F F F 22 440-459 127 127 127 216 216 216 F F F 23 460-479 128 128 128 217 217 217 F F F 24 480-499 126 126 126 214 214 214 F F F 25 500-519 126 126 126 214 214 214 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 125 125 125 211 211 211 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 128, Max flush: 219 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 119 119 119 200 200 200 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 123 123 123 207 207 207 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 129 129 129 220 220 220 F F F 8 160-179 128 128 128 219 219 219 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 128 128 128 219 219 219 F F F 11 220-239 129 129 129 220 220 220 F F F 12 240-259 127 127 127 216 216 216 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 129 129 129 220 220 220 F F F 15 300-319 129 129 129 220 220 220 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 121 121 121 203 203 203 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 128 128 128 219 219 219 F F F 22 440-459 127 127 127 216 216 216 F F F 23 460-479 128 128 128 217 217 217 F F F 24 480-499 126 126 126 214 214 214 F F F 25 500-519 126 126 126 214 214 214 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 125 125 125 211 211 211 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 129, Max flush: 220 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.525095361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.570392684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.571050582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.57108849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.571103459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.571125928Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} {"msg":"Crucible 10729632-3d73-4423-8b58-114c903ecefc has session id: 932956f8-bbb3-47e6-8c0f-e6eef9565e6f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.571651872Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Upstairs opts: Upstairs UUID: 10729632-3d73-4423-8b58-114c903ecefc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.57169724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Crucible stats registered with UUID: 10729632-3d73-4423-8b58-114c903ecefc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.571721969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602110554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602139873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} {"msg":"10729632-3d73-4423-8b58-114c903ecefc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602211689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602235408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602258657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602273066Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602286706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602300195Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602313695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602326904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602341063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602360972Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41640)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602499115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44671)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602541293Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44245)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.602572712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603207711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603237749Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603253458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603270697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603401981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.60342345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603939284Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603969823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.603985342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604513576Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604554044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604573213Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604587113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604601452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604615171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604627481Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.60464067Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[0]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 220, 219, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604654789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[0]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 129, 128, 128, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604672028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604686658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[1]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 220, 219, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604700527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[1]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 129, 128, 128, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604714206Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604728186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[2]R flush_numbers[0..12]: [215, 211, 207, 200, 211, 207, 201, 220, 219, 219, 219, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604746215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[2]R generation[0..12]: [126, 125, 123, 119, 125, 123, 120, 129, 128, 128, 128, 129]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:09:09.604772823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604801902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604821411Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604835311Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.6048485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604861829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"":"downstairs","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604875478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604887718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"0","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604901397Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"1","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604918106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"client":"2","":"downstairs","session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604931936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:09:09.604944425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"10729632-3d73-4423-8b58-114c903ecefc is now active with session: 932956f8-bbb3-47e6-8c0f-e6eef9565e6f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604960584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604982813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702,"session_id":"932956f8-bbb3-47e6-8c0f-e6eef9565e6f"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:09.604996583Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 8192 data: 15 19 02/30 Write block 112 len 24576 data: 17 12 11 11 11 10 03/30 Read block 480 len 12288 04/30 Write block 133 len 36864 data: 16 14 13 14 13 16 14 13 16 05/30 Flush 06/30 Write block 390 len 16384 data: 9 10 12 11 07/30 Write block 509 len 28672 data: 9 8 9 9 8 8 10 08/30 Flush 09/30 Write block 56 len 24576 data: 10 8 9 8 7 6 10/30 Write block 201 len 8192 data: 11 8 11/30 Write block 244 len 40960 data: 12 9 9 10 8 9 10 9 9 10 12/30 Read block 302 len 20480 13/30 Flush 14/30 Write block 280 len 28672 data: 12 11 9 9 8 8 9 15/30 Read block 472 len 40960 16/30 Write block 563 len 12288 data: 12 10 12 17/30 Flush 18/30 Write block 576 len 32768 data: 11 12 11 11 9 11 11 11 19/30 Write block 517 len 8192 data: 10 13 20/30 Flush 21/30 Flush 22/30 Write block 458 len 24576 data: 8 9 13 12 11 14 23/30 Read block 223 len 8192 24/30 Write block 422 len 28672 data: 10 8 7 7 8 6 7 25/30 Read block 410 len 4096 26/30 Read block 352 len 8192 27/30 Write block 34 len 32768 data: 11 14 14 14 13 11 12 15 28/30 Write block 481 len 16384 data: 13 13 11 10 29/30 Write block 405 len 12288 data: 7 8 11 30/30 Write block 161 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:09.622314981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 125 125 125 211 211 211 T T T 2 040-059 130 130 130 223 223 223 T T T 3 060-079 130 130 130 223 223 223 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 130 130 130 221 221 221 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 128 128 128 219 219 219 T T T 9 180-199 128 128 128 219 219 219 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 129 129 129 220 220 220 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 130 130 130 224 224 224 F F F 15 300-319 129 129 129 220 220 220 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 124 124 124 209 209 209 T T T 21 420-439 128 128 128 219 219 219 T T T 22 440-459 130 130 130 221 221 221 T T T 23 460-479 128 128 128 217 217 217 T T T 24 480-499 126 126 126 214 214 214 T T T 25 500-519 130 130 130 225 225 225 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 130 130 130 225 225 225 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 130, Max flush: 225 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 131 131 131 226 226 226 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 130 130 130 221 221 221 F F F 6 120-139 130 130 130 221 221 221 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 131 131 131 226 226 226 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 129 129 129 220 220 220 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 130 130 130 224 224 224 F F F 15 300-319 129 129 129 220 220 220 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 131 131 131 226 226 226 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 131 131 131 226 226 226 F F F 25 500-519 130 130 130 225 225 225 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 130 130 130 225 225 225 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 131, Max flush: 226 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.909191203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:13.952734552Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.953144852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.985437994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986024965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986060593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986075213Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986089842Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} {"msg":"Crucible c1d3c64d-817d-4bd0-9872-b4759fb6c2fc has session id: 32453dd0-f247-4cb5-a8a3-aed1b19695dc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986516601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Upstairs opts: Upstairs UUID: c1d3c64d-817d-4bd0-9872-b4759fb6c2fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986563939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Crucible stats registered with UUID: c1d3c64d-817d-4bd0-9872-b4759fb6c2fc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:13.986582188Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.016878368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.016911996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} {"msg":"c1d3c64d-817d-4bd0-9872-b4759fb6c2fc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.016938415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.016955194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.016986603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017005692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017019871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01703359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01704758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017061119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017075219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017089238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33150)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017284688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47864)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017323746Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45741)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017393303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017947616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.017984004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.018003683Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.018019912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.018037791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01806093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.018630742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01866149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.0186774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019246772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01927486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01928981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019364136Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019398844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019415433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019428363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019441912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[0]R flush_numbers[0..12]: [215, 226, 226, 223, 211, 221, 221, 221, 226, 219, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019455862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[0]R generation[0..12]: [126, 131, 131, 130, 125, 130, 130, 130, 131, 128, 130, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019469941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01948691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[1]R flush_numbers[0..12]: [215, 226, 226, 223, 211, 221, 221, 221, 226, 219, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019501299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[1]R generation[0..12]: [126, 131, 131, 130, 125, 130, 130, 130, 131, 128, 130, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019514988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019528728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[2]R flush_numbers[0..12]: [215, 226, 226, 223, 211, 221, 221, 221, 226, 219, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019542437Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[2]R generation[0..12]: [126, 131, 131, 130, 125, 130, 130, 130, 131, 128, 130, 129]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019556297Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019577345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Max found gen is 132","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-26T22:09:14.019611254Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019636493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019651522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019664601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"":"downstairs","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019677971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.01969017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"0","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019703589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"1","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019716659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"client":"2","":"downstairs","session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019732478Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019744877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"c1d3c64d-817d-4bd0-9872-b4759fb6c2fc is now active with session: 32453dd0-f247-4cb5-a8a3-aed1b19695dc","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019757817Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-11-26T22:09:14.019773386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710,"session_id":"32453dd0-f247-4cb5-a8a3-aed1b19695dc"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:14.019785905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 32768 data: 10 9 10 10 10 11 10 11 02/30 Read block 546 len 4096 03/30 Write block 164 len 24576 data: 11 9 10 11 13 13 04/30 Read block 554 len 28672 05/30 Write block 162 len 24576 data: 10 7 12 10 11 12 06/30 Write block 124 len 40960 data: 7 7 8 11 10 13 15 16 14 17 07/30 Read block 299 len 8192 08/30 Read block 83 len 36864 09/30 Write block 305 len 32768 data: 7 13 15 15 11 13 14 14 10/30 Write block 559 len 32768 data: 11 11 11 11 13 11 13 14 11/30 Read block 20 len 4096 12/30 Read block 250 len 4096 13/30 Read block 289 len 36864 14/30 Read block 399 len 24576 15/30 Read block 217 len 28672 16/30 Read block 141 len 32768 17/30 Flush 18/30 Write block 353 len 4096 data: 16 19/30 Write block 573 len 12288 data: 12 14 11 20/30 Write block 303 len 4096 data: 11 21/30 Read block 146 len 8192 22/30 Write block 525 len 24576 data: 14 14 13 11 11 11 23/30 Read block 361 len 36864 24/30 Write block 43 len 36864 data: 17 20 15 11 11 10 10 10 9 25/30 Read block 422 len 40960 26/30 Read block 327 len 24576 27/30 Read block 390 len 36864 28/30 Read block 123 len 28672 29/30 Write block 284 len 24576 data: 9 9 10 13 14 12 30/30 Read block 262 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 12 13 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:14.040163323Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 131 131 131 226 226 226 T T T 3 060-079 130 130 130 223 223 223 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 130 130 130 221 221 221 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 132 132 132 227 227 227 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 132 132 132 227 227 227 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 130 130 130 224 224 224 T T T 15 300-319 132 132 132 227 227 227 T T T 16 320-339 122 122 122 206 206 206 F F F 17 340-359 119 119 119 200 200 200 T T T 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 131 131 131 226 226 226 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 131 131 131 226 226 226 F F F 25 500-519 130 130 130 225 225 225 F F F 26 520-539 127 127 127 216 216 216 T T T 27 540-559 132 132 132 227 227 227 F F F 28 560-579 132 132 132 227 227 227 T T T 29 580-599 130 130 130 225 225 225 F F F Max gen: 132, Max flush: 227 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 133 133 133 228 228 228 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 125 125 125 211 211 211 F F F 5 100-119 130 130 130 221 221 221 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 132 132 132 227 227 227 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 132 132 132 227 227 227 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 124 124 124 210 210 210 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 131 131 131 226 226 226 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 131 131 131 226 226 226 F F F 25 500-519 130 130 130 225 225 225 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.349038346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.394233574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.39491155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.394949368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.394963488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.394977777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} {"msg":"Crucible 65c86462-9788-4a0c-96fa-908c913cc4db has session id: cd6834cf-39ab-431b-a175-731639354353","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.395485792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Upstairs opts: Upstairs UUID: 65c86462-9788-4a0c-96fa-908c913cc4db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.39553223Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Crucible stats registered with UUID: 65c86462-9788-4a0c-96fa-908c913cc4db","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.395556429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426696997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426723826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} {"msg":"65c86462-9788-4a0c-96fa-908c913cc4db active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426784713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426802782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426828551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.42685663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426956625Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.426990323Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.42724911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427282878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427307857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427331896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_connection connected from Ok(127.0.0.1:43495)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427356855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_connection connected from Ok(127.0.0.1:39120)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427381264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_connection connected from Ok(127.0.0.1:39760)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427404903Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.427810032Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.42785534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428020772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428053891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428074439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428101258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428572495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428618513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.428832962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429092289Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429119278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429134278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429146757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429160586Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429174145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429350827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429380245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[0]R flush_numbers[0..12]: [215, 226, 228, 223, 211, 221, 227, 221, 227, 219, 223, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429397374Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[0]R generation[0..12]: [126, 131, 133, 130, 125, 130, 132, 130, 132, 128, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429412044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429426283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[1]R flush_numbers[0..12]: [215, 226, 228, 223, 211, 221, 227, 221, 227, 219, 223, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429440212Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[1]R generation[0..12]: [126, 131, 133, 130, 125, 130, 132, 130, 132, 128, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429457911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429472711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[2]R flush_numbers[0..12]: [215, 226, 228, 223, 211, 221, 227, 221, 227, 219, 223, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.42948662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[2]R generation[0..12]: [126, 131, 133, 130, 125, 130, 132, 130, 132, 128, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.42950034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429514189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429527248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429540618Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429553727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429576736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"":"downstairs","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429603124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429627113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"0","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429652762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"1","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429678401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"client":"2","":"downstairs","session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429706859Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:09:18.429733058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"65c86462-9788-4a0c-96fa-908c913cc4db is now active with session: cd6834cf-39ab-431b-a175-731639354353","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429760217Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429775636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718,"session_id":"cd6834cf-39ab-431b-a175-731639354353"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:18.429788365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 421 len 8192 data: 11 11 03/30 Read block 567 len 40960 04/30 Flush 05/30 Write block 202 len 36864 data: 9 10 11 9 8 8 6 7 7 06/30 Read block 279 len 16384 07/30 Read block 102 len 36864 08/30 Read block 66 len 24576 09/30 Flush 10/30 Write block 278 len 4096 data: 12 11/30 Read block 200 len 16384 12/30 Write block 516 len 16384 data: 11 11 14 14 13/30 Read block 491 len 24576 14/30 Write block 156 len 32768 data: 9 8 10 11 10 12 11 8 15/30 Write block 102 len 16384 data: 11 12 12 14 16/30 Read block 511 len 4096 17/30 Write block 156 len 20480 data: 10 9 11 12 11 18/30 Write block 99 len 12288 data: 7 6 9 19/30 Read block 42 len 32768 20/30 Read block 492 len 24576 21/30 Read block 365 len 8192 22/30 Read block 95 len 4096 23/30 Write block 488 len 4096 data: 9 24/30 Read block 315 len 20480 25/30 Read block 110 len 20480 26/30 Read block 205 len 36864 27/30 Read block 595 len 16384 28/30 Write block 157 len 4096 data: 10 29/30 Write block 150 len 32768 data: 7 7 4 4 7 9 11 11 30/30 Write block 570 len 24576 data: 13 12 11 13 15 12 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:18.449881547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 133 133 133 228 228 228 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 125 125 125 211 211 211 T T T 5 100-119 130 130 130 221 221 221 T T T 6 120-139 132 132 132 227 227 227 F F F 7 140-159 130 130 130 221 221 221 T T T 8 160-179 132 132 132 227 227 227 T T T 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 132 132 132 227 227 227 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 124 124 124 210 210 210 T T T 14 280-299 133 133 133 228 228 228 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 134 134 134 230 230 230 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 131 131 131 226 226 226 T T T 25 500-519 130 130 130 225 225 225 T T T 26 520-539 133 133 133 228 228 228 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 133 133 133 228 228 228 T T T 29 580-599 130 130 130 225 225 225 F F F Max gen: 134, Max flush: 231 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 133 133 133 228 228 228 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 135 135 135 232 232 232 F F F 8 160-179 135 135 135 232 232 232 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 132 132 132 227 227 227 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 135 135 135 232 232 232 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 133 133 133 228 228 228 F F F 16 320-339 122 122 122 206 206 206 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 134 134 134 230 230 230 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 135 135 135 232 232 232 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 135 135 135 232 232 232 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 135, Max flush: 232 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.734767109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.779805364Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.780651042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.78068978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.78070408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.780718579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} {"msg":"Crucible 4c61aa28-50b5-444d-9605-75ebbc4a6c75 has session id: 76b6a94e-613e-4a22-aa12-aff0dd15ef77","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.781160488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Upstairs opts: Upstairs UUID: 4c61aa28-50b5-444d-9605-75ebbc4a6c75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.781200166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Crucible stats registered with UUID: 4c61aa28-50b5-444d-9605-75ebbc4a6c75","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.781217065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811656778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811683417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} {"msg":"4c61aa28-50b5-444d-9605-75ebbc4a6c75 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811760273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811779392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811802051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81181623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811834559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811848728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811862508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811875837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811889697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.811903206Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_connection connected from Ok(127.0.0.1:51737)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812064008Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_connection connected from Ok(127.0.0.1:33530)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812104926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_connection connected from Ok(127.0.0.1:35939)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812253988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81283888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812867149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812882318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812896227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.812985553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.813015691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.813462779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.813529956Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.813570624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81405382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814083969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814100068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814167685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814187904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814202163Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814214812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814228442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[0]R flush_numbers[0..12]: [215, 226, 228, 223, 232, 232, 227, 232, 232, 219, 231, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814242371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[0]R generation[0..12]: [126, 131, 133, 130, 135, 135, 132, 135, 135, 128, 134, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81425636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814270549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[1]R flush_numbers[0..12]: [215, 226, 228, 223, 232, 232, 227, 232, 232, 219, 231, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814284489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[1]R generation[0..12]: [126, 131, 133, 130, 135, 135, 132, 135, 135, 128, 134, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814298278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814315517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[2]R flush_numbers[0..12]: [215, 226, 228, 223, 232, 232, 227, 232, 232, 219, 231, 227]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814329857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[2]R generation[0..12]: [126, 131, 133, 130, 135, 135, 132, 135, 135, 128, 134, 132]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814343776Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814357885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814371055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814384304Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814397333Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814421652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"":"downstairs","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814435061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814447111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"0","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81446061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"1","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.81447375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"client":"2","":"downstairs","session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814486939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814499278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"4c61aa28-50b5-444d-9605-75ebbc4a6c75 is now active with session: 76b6a94e-613e-4a22-aa12-aff0dd15ef77","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:09:22.814511968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814531227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726,"session_id":"76b6a94e-613e-4a22-aa12-aff0dd15ef77"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:22.814543576Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 250 len 20480 data: 11 10 10 11 7 02/30 Flush 03/30 Write block 439 len 4096 data: 18 04/30 Read block 288 len 24576 05/30 Write block 315 len 36864 data: 13 10 7 8 8 7 8 9 9 06/30 Read block 538 len 36864 07/30 Read block 344 len 32768 08/30 Flush 09/30 Flush 10/30 Write block 423 len 24576 data: 9 8 8 9 7 8 11/30 Read block 287 len 16384 12/30 Write block 236 len 40960 data: 12 12 10 10 10 13 14 13 13 10 13/30 Flush 14/30 Write block 45 len 4096 data: 16 15/30 Read block 16 len 12288 16/30 Read block 51 len 4096 17/30 Flush 18/30 Write block 455 len 20480 data: 6 7 8 9 10 19/30 Write block 569 len 20480 data: 14 14 13 12 14 20/30 Read block 561 len 28672 21/30 Flush 22/30 Flush 23/30 Write block 148 len 20480 data: 7 6 8 8 5 24/30 Read block 89 len 28672 25/30 Write block 171 len 4096 data: 13 26/30 Read block 541 len 36864 27/30 Write block 513 len 8192 data: 9 9 28/30 Read block 502 len 12288 29/30 Write block 572 len 20480 data: 13 15 16 13 12 30/30 Write block 227 len 24576 data: 11 10 11 11 11 12 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:22.832903073Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 135 135 135 232 232 232 T T T 8 160-179 135 135 135 232 232 232 T T T 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 136 136 136 236 236 236 T T T 12 240-259 136 136 136 236 236 236 F F F 13 260-279 135 135 135 232 232 232 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 135 135 135 232 232 232 T T T 26 520-539 133 133 133 228 228 228 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 136 136 136 238 238 238 T T T 29 580-599 130 130 130 225 225 225 F F F Max gen: 136, Max flush: 238 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 215 215 215 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 130 130 130 223 223 223 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 137 137 137 239 239 239 F F F 8 160-179 137 137 137 239 239 239 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 137 137 137 239 239 239 F F F 12 240-259 136 136 136 236 236 236 F F F 13 260-279 135 135 135 232 232 232 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 122 122 122 205 205 205 F F F 19 380-399 130 130 130 222 222 222 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 137 137 137 239 239 239 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 137, Max flush: 239 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.1046574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.149629169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150293016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150330374Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150349083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150364093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} {"msg":"Crucible 6ac95182-da68-4825-a3a0-ca46a8279b21 has session id: 13bc3bea-5ff2-47ae-b494-4826e30d22f8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.15081222Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Upstairs opts: Upstairs UUID: 6ac95182-da68-4825-a3a0-ca46a8279b21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150858678Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Crucible stats registered with UUID: 6ac95182-da68-4825-a3a0-ca46a8279b21","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.150884777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182060624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182089433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} {"msg":"6ac95182-da68-4825-a3a0-ca46a8279b21 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.18214428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182172749Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182198587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182214527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182240275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182267394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182292333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182309052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182331271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182360499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48900)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182614727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41085)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182657495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62204)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.182688143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183250065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183279364Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183294843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183309013Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183322702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.183354271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184022307Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184076595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184108913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184604449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184635257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184650137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184714744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184735863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184749732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184761961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184775801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[0]R flush_numbers[0..12]: [215, 226, 237, 223, 232, 232, 227, 239, 239, 219, 231, 239]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.18478992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[0]R generation[0..12]: [126, 131, 136, 130, 135, 135, 132, 137, 137, 128, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184804039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184817939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[1]R flush_numbers[0..12]: [215, 226, 237, 223, 232, 232, 227, 239, 239, 219, 231, 239]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184832048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[1]R generation[0..12]: [126, 131, 136, 130, 135, 135, 132, 137, 137, 128, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184845777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184859766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[2]R flush_numbers[0..12]: [215, 226, 237, 223, 232, 232, 227, 239, 239, 219, 231, 239]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184873656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[2]R generation[0..12]: [126, 131, 136, 130, 135, 135, 132, 137, 137, 128, 134, 137]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184887205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184903794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184917123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.184930513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairsWait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185012029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185044667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"":"downstairs","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185062776Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185076096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"0","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185098095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"1","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185125283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"client":"2","":"downstairs","session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185153092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185168381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"6ac95182-da68-4825-a3a0-ca46a8279b21 is now active with session: 13bc3bea-5ff2-47ae-b494-4826e30d22f8","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.18518554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185208719Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734,"session_id":"13bc3bea-5ff2-47ae-b494-4826e30d22f8"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:27.185233728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 8192 data: 8 7 02/30 Write block 39 len 8192 data: 12 13 03/30 Write block 387 len 4096 data: 9 04/30 Write block 268 len 16384 data: 9 11 11 9 05/30 Write block 225 len 4096 data: 8 06/30 Read block 29 len 12288 07/30 Write block 285 len 40960 data: 10 11 14 15 13 13 9 9 7 9 08/30 Read block 211 len 28672 09/30 Write block 147 len 4096 data: 10 10/30 Read block 454 len 16384 11/30 Write block 58 len 20480 data: 10 9 8 7 9 12/30 Read block 227 len 24576 13/30 Read block 113 len 32768 14/30 Read block 299 len 20480 15/30 Write block 360 len 4096 data: 9 16/30 Write block 5 len 24576 data: 10 10 11 10 13 13 17/30 Write block 233 len 36864 data: 11 12 12 13 13 11 11 11 14 18/30 Read block 527 len 36864 19/30 Read block 381 len 24576 20/30 Read block 325 len 12288 21/30 Read block 43 len 16384 22/30 Read block 334 len 40960 23/30 Write block 541 len 24576 data: 10 11 7 8 11 14 24/30 Flush 25/30 Write block 132 len 24576 data: 15 18 15 14 15 14 26/30 Read block 200 len 8192 27/30 Write block 224 len 28672 data: 7 9 12 12 11 12 12 28/30 Read block 463 len 16384 29/30 Read block 230 len 32768 30/30 Write block 220 len 32768 data: 9 9 7 6 8 10 13 13 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:27.204553578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 240 240 240 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 132 132 132 227 227 227 T T T 7 140-159 138 138 138 240 240 240 F F F 8 160-179 137 137 137 239 239 239 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 138 138 138 240 240 240 T T T 12 240-259 138 138 138 240 240 240 F F F 13 260-279 138 138 138 240 240 240 F F F 14 280-299 138 138 138 240 240 240 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 138 138 138 240 240 240 F F F 20 400-419 138 138 138 240 240 240 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 137 137 137 239 239 239 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 138 138 138 240 240 240 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 138, Max flush: 240 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 240 240 240 F F F 1 020-039 138 138 138 240 240 240 F F F 2 040-059 138 138 138 240 240 240 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 139 139 139 241 241 241 F F F 7 140-159 138 138 138 240 240 240 F F F 8 160-179 137 137 137 239 239 239 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 139 139 139 241 241 241 F F F 12 240-259 138 138 138 240 240 240 F F F 13 260-279 138 138 138 240 240 240 F F F 14 280-299 138 138 138 240 240 240 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 138 138 138 240 240 240 F F F 20 400-419 138 138 138 240 240 240 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 131 131 131 226 226 226 F F F 24 480-499 135 135 135 232 232 232 F F F 25 500-519 137 137 137 239 239 239 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 138 138 138 240 240 240 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 130 130 130 225 225 225 F F F Max gen: 139, Max flush: 241 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.456286249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:31.49979316Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.500161182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.532438855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.532989738Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.533022546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.533036256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.533050615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} {"msg":"Crucible aa5a3422-2fc8-44c7-a5f5-8ef55db3c1a1 has session id: d2c02ac5-a74f-4daf-803b-c20224f732f9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.533485263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Upstairs opts: Upstairs UUID: aa5a3422-2fc8-44c7-a5f5-8ef55db3c1a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.533531941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Crucible stats registered with UUID: aa5a3422-2fc8-44c7-a5f5-8ef55db3c1a1","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.5335595Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.563873039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.563905128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} {"msg":"aa5a3422-2fc8-44c7-a5f5-8ef55db3c1a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.563973555Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.563996403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564025142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564042881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.56406862Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564164395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564205713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564237442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564269Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564302128Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46903)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564341216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63436)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564369695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41227)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.564397314Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565131847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565169856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565200064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565229243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565256021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565271321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565839123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565879561Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.565911969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566396265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566421554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566435793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.56651505Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566548018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566564037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566576587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566591106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 240, 240, 232, 232, 241, 240, 239, 219, 231, 241]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566605335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[0]R generation[0..12]: [138, 138, 138, 138, 135, 135, 139, 138, 137, 128, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566619354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566633574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 240, 240, 232, 232, 241, 240, 239, 219, 231, 241]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566647533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[1]R generation[0..12]: [138, 138, 138, 138, 135, 135, 139, 138, 137, 128, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566661192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566675291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 240, 240, 232, 232, 241, 240, 239, 219, 231, 241]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566689091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[2]R generation[0..12]: [138, 138, 138, 138, 135, 135, 139, 138, 137, 128, 134, 139]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.5667028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:09:31.566717029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566744698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566771147Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566813655Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566840174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"":"downstairs","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566864752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566878661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"0","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566895941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"1","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.56690982Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"client":"2","":"downstairs","session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566923239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566935579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"aa5a3422-2fc8-44c7-a5f5-8ef55db3c1a1 is now active with session: d2c02ac5-a74f-4daf-803b-c20224f732f9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566948508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566960428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742,"session_id":"d2c02ac5-a74f-4daf-803b-c20224f732f9"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:31.566972997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 28672 data: 6 8 8 12 15 15 15 02/30 Read block 244 len 28672 03/30 Write block 517 len 36864 data: 12 15 15 14 12 12 13 15 15 04/30 Write block 363 len 32768 data: 8 7 8 9 8 8 9 12 05/30 Write block 144 len 36864 data: 12 11 8 11 8 7 9 9 6 06/30 Write block 268 len 8192 data: 10 12 07/30 Write block 29 len 28672 data: 5 7 7 9 9 13 16 08/30 Flush 09/30 Write block 23 len 8192 data: 12 12 10/30 Flush 11/30 Write block 482 len 16384 data: 14 12 11 8 12/30 Write block 40 len 36864 data: 14 16 17 18 21 17 12 12 11 13/30 Read block 558 len 36864 14/30 Read block 546 len 28672 15/30 Read block 195 len 4096 16/30 Write block 472 len 20480 data: 9 9 9 6 6 17/30 Read block 101 len 8192 18/30 Write block 483 len 40960 data: 13 12 9 9 8 10 10 10 11 11 19/30 Write block 549 len 28672 data: 9 7 6 4 4 5 6 20/30 Flush 21/30 Write block 476 len 36864 data: 7 7 8 8 10 14 15 14 13 22/30 Write block 233 len 4096 data: 12 23/30 Write block 353 len 20480 data: 17 13 11 9 8 24/30 Read block 6 len 20480 25/30 Read block 579 len 24576 26/30 Read block 152 len 36864 27/30 Read block 96 len 4096 28/30 Write block 15 len 8192 data: 11 12 29/30 Write block 593 len 12288 data: 10 9 10 30/30 Read block 71 len 20480 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:31.585338934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 240 240 240 T T T 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 139 139 139 241 241 241 F F F 7 140-159 140 140 140 242 242 242 F F F 8 160-179 137 137 137 239 239 239 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 139 139 139 241 241 241 T T T 12 240-259 138 138 138 240 240 240 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 138 138 138 240 240 240 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 133 133 133 228 228 228 T T T 18 360-379 140 140 140 242 242 242 F F F 19 380-399 138 138 138 240 240 240 F F F 20 400-419 138 138 138 240 240 240 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 140 140 140 244 244 244 T T T 24 480-499 140 140 140 244 244 244 T T T 25 500-519 140 140 140 242 242 242 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 140 140 140 244 244 244 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 130 130 130 225 225 225 T T T Max gen: 140, Max flush: 244 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 135 135 135 232 232 232 F F F 5 100-119 135 135 135 232 232 232 F F F 6 120-139 139 139 139 241 241 241 F F F 7 140-159 140 140 140 242 242 242 F F F 8 160-179 137 137 137 239 239 239 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 134 134 134 231 231 231 F F F 11 220-239 141 141 141 245 245 245 F F F 12 240-259 138 138 138 240 240 240 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 138 138 138 240 240 240 F F F 15 300-319 136 136 136 234 234 234 F F F 16 320-339 136 136 136 234 234 234 F F F 17 340-359 141 141 141 245 245 245 F F F 18 360-379 140 140 140 242 242 242 F F F 19 380-399 138 138 138 240 240 240 F F F 20 400-419 138 138 138 240 240 240 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 141 141 141 245 245 245 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 140 140 140 242 242 242 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 140 140 140 244 244 244 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 141 141 141 245 245 245 F F F Max gen: 141, Max flush: 245 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.861668324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.906788075Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.907433104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.907471062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.907485951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.90750031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} {"msg":"Crucible 185ba72c-0d57-4287-8c0d-9c1b99bdfbc4 has session id: 1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.907943019Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Upstairs opts: Upstairs UUID: 185ba72c-0d57-4287-8c0d-9c1b99bdfbc4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.907988646Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Crucible stats registered with UUID: 185ba72c-0d57-4287-8c0d-9c1b99bdfbc4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.908025324Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939278578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939313816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} {"msg":"185ba72c-0d57-4287-8c0d-9c1b99bdfbc4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939348714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939365754Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939393072Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939410681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.93942778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.93944474Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939472278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939505747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939542775Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939576153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64136)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.939610371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:52499)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.93964486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58429)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.93983688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.940399593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.940427091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.94044219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.94045616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.94066341Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.940707068Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941095449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941125387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941140746Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941649611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.9416742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941688339Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941756896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941776505Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941794134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941806914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941820343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[0]R flush_numbers[0..12]: [245, 243, 244, 240, 232, 232, 241, 242, 239, 219, 231, 245]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941848092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[0]R generation[0..12]: [141, 140, 140, 138, 135, 135, 139, 140, 137, 128, 134, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941862421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.94187638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[1]R flush_numbers[0..12]: [245, 243, 244, 240, 232, 232, 241, 242, 239, 219, 231, 245]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941890239Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[1]R generation[0..12]: [141, 140, 140, 138, 135, 135, 139, 140, 137, 128, 134, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941903969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941917968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[2]R flush_numbers[0..12]: [245, 243, 244, 240, 232, 232, 241, 242, 239, 219, 231, 245]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941931707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[2]R generation[0..12]: [141, 140, 140, 138, 135, 135, 139, 140, 137, 128, 134, 141]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941945617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941959536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941972475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.941985645Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942004584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942030043Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"":"downstairs","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942045712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942058031Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"0","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.94208088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"1","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942094249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"client":"2","":"downstairs","session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942107639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942120158Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"185ba72c-0d57-4287-8c0d-9c1b99bdfbc4 is now active with session: 1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942133087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942145077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750,"session_id":"1d8b2e6a-aa98-4c13-8c89-7502ae1c4cf3"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:35.942157396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 24576 data: 11 11 12 12 12 10 02/30 Read block 233 len 12288 03/30 Read block 254 len 28672 04/30 Read block 233 len 40960 05/30 Write block 411 len 36864 data: 9 9 8 11 10 11 12 12 10 06/30 Flush 07/30 Write block 218 len 36864 data: 12 12 10 10 8 7 9 11 14 08/30 Write block 128 len 4096 data: 11 09/30 Read block 125 len 24576 10/30 Write block 263 len 8192 data: 13 12 11/30 Write block 90 len 32768 data: 12 12 12 9 9 11 13 9 12/30 Write block 511 len 28672 data: 10 10 10 10 11 12 13 13/30 Write block 92 len 40960 data: 13 10 10 12 14 10 9 8 7 10 14/30 Write block 322 len 36864 data: 10 10 11 11 9 9 9 7 9 15/30 Write block 393 len 8192 data: 12 9 16/30 Read block 219 len 12288 17/30 Write block 254 len 36864 data: 8 7 8 9 9 7 12 11 12 18/30 Read block 556 len 24576 19/30 Write block 156 len 20480 data: 12 12 12 13 12 20/30 Read block 495 len 28672 21/30 Write block 357 len 20480 data: 9 11 10 10 8 22/30 Read block 334 len 24576 23/30 Read block 522 len 4096 24/30 Flush 25/30 Write block 230 len 12288 data: 13 12 13 26/30 Flush 27/30 Write block 146 len 24576 data: 9 12 9 8 10 10 28/30 Write block 119 len 32768 data: 10 9 6 6 7 8 8 9 29/30 Read block 496 len 24576 30/30 Read block 41 len 32768 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:35.963126865Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 142 142 142 247 247 247 T T T 6 120-139 142 142 142 247 247 247 T T T 7 140-159 142 142 142 247 247 247 T T T 8 160-179 142 142 142 247 247 247 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 142 142 142 247 247 247 F F F 11 220-239 142 142 142 248 248 248 F F F 12 240-259 142 142 142 247 247 247 F F F 13 260-279 142 142 142 247 247 247 F F F 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 142 142 142 247 247 247 F F F 18 360-379 142 142 142 247 247 247 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 246 246 246 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 141 141 141 245 245 245 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 142 142 142 247 247 247 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 140 140 140 244 244 244 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 141 141 141 245 245 245 F F F Max gen: 142, Max flush: 248 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 140 140 140 244 244 244 F F F 3 060-079 138 138 138 240 240 240 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 143 143 143 249 249 249 F F F 6 120-139 143 143 143 249 249 249 F F F 7 140-159 143 143 143 249 249 249 F F F 8 160-179 142 142 142 247 247 247 F F F 9 180-199 128 128 128 219 219 219 F F F 10 200-219 142 142 142 247 247 247 F F F 11 220-239 142 142 142 248 248 248 F F F 12 240-259 142 142 142 247 247 247 F F F 13 260-279 142 142 142 247 247 247 F F F 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 142 142 142 247 247 247 F F F 18 360-379 142 142 142 247 247 247 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 246 246 246 F F F 21 420-439 136 136 136 236 236 236 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 141 141 141 245 245 245 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 142 142 142 247 247 247 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 140 140 140 244 244 244 F F F 28 560-579 137 137 137 239 239 239 F F F 29 580-599 141 141 141 245 245 245 F F F Max gen: 143, Max flush: 249 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.220837241Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.266000351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.266606301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.266655299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.266680647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.266704706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} {"msg":"Crucible a4b877c4-23f0-43c0-ada7-e35208366c3f has session id: 7f86e170-b6f0-49cf-8999-c492bcb3598c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.267184423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Upstairs opts: Upstairs UUID: a4b877c4-23f0-43c0-ada7-e35208366c3f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.26722972Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Crucible stats registered with UUID: a4b877c4-23f0-43c0-ada7-e35208366c3f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.267254949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297504342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.2975388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} {"msg":"a4b877c4-23f0-43c0-ada7-e35208366c3f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297573528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297591127Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297618926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297636845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297653664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297670434Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297686933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297712392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.29774591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297779528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42313)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297816267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39776)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297850735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48539)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.297885513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.298505923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.298539001Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.29856869Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.298665465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.298683014Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.298697103Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299236077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299270055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299357911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299846107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299885265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299909424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299931423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.299954931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.29998025Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300001529Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300024338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[0]R flush_numbers[0..12]: [245, 243, 244, 240, 247, 249, 249, 249, 247, 219, 247, 248]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300049157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[0]R generation[0..12]: [141, 140, 140, 138, 142, 143, 143, 143, 142, 128, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300079895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300108574Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[1]R flush_numbers[0..12]: [245, 243, 244, 240, 247, 249, 249, 249, 247, 219, 247, 248]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300134993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[1]R generation[0..12]: [141, 140, 140, 138, 142, 143, 143, 143, 142, 128, 142, 142]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-11-26T22:09:40.300172421Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300203829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[2]R flush_numbers[0..12]: [245, 243, 244, 240, 247, 249, 249, 249, 247, 219, 247, 248]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300233468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[2]R generation[0..12]: [141, 140, 140, 138, 142, 143, 143, 143, 142, 128, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300263026Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300292615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300320443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300350772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.30037889Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300406729Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"":"downstairs","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300422248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300434558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"0","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300450917Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"1","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300464556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"client":"2","":"downstairs","session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300477786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300490265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"a4b877c4-23f0-43c0-ada7-e35208366c3f is now active with session: 7f86e170-b6f0-49cf-8999-c492bcb3598c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300503134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300515064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758,"session_id":"7f86e170-b6f0-49cf-8999-c492bcb3598c"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:40.300527473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 36864 data: 13 12 11 11 10 12 11 13 9 02/30 Write block 44 len 28672 data: 22 18 13 14 13 12 12 03/30 Write block 462 len 24576 data: 12 15 13 14 15 13 04/30 Read block 451 len 28672 05/30 Flush 06/30 Write block 373 len 16384 data: 10 9 11 13 07/30 Write block 572 len 8192 data: 14 16 08/30 Write block 214 len 24576 data: 10 12 14 10 13 13 09/30 Write block 580 len 16384 data: 10 12 12 12 10/30 Read block 391 len 28672 11/30 Read block 259 len 16384 12/30 Read block 58 len 12288 13/30 Read block 436 len 28672 14/30 Read block 392 len 28672 15/30 Write block 120 len 16384 data: 10 7 7 8 16/30 Write block 227 len 4096 data: 14 17/30 Write block 200 len 32768 data: 10 12 10 11 12 10 9 9 18/30 Write block 425 len 4096 data: 9 19/30 Read block 281 len 4096 20/30 Write block 264 len 20480 data: 13 11 9 10 11 21/30 Read block 564 len 4096 22/30 Write block 344 len 32768 data: 7 10 8 8 12 13 12 13 23/30 Write block 474 len 24576 data: 10 7 8 8 9 9 24/30 Write block 54 len 40960 data: 14 10 11 9 11 10 9 8 10 10 25/30 Write block 195 len 32768 data: 10 12 11 10 10 11 13 11 26/30 Read block 240 len 12288 27/30 Read block 13 len 16384 28/30 Write block 553 len 20480 data: 5 6 7 7 8 29/30 Write block 568 len 40960 data: 14 15 15 14 15 17 17 14 13 13 30/30 Read block 252 len 20480 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 25 24 23 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:40.318634003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 144 144 144 250 250 250 T T T 3 060-079 138 138 138 240 240 240 T T T 4 080-099 142 142 142 247 247 247 F F F 5 100-119 143 143 143 249 249 249 F F F 6 120-139 143 143 143 249 249 249 T T T 7 140-159 143 143 143 249 249 249 F F F 8 160-179 142 142 142 247 247 247 F F F 9 180-199 128 128 128 219 219 219 T T T 10 200-219 142 142 142 247 247 247 T T T 11 220-239 142 142 142 248 248 248 T T T 12 240-259 142 142 142 247 247 247 F F F 13 260-279 142 142 142 247 247 247 T T T 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 142 142 142 247 247 247 T T T 18 360-379 142 142 142 247 247 247 T T T 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 246 246 246 F F F 21 420-439 136 136 136 236 236 236 T T T 22 440-459 136 136 136 238 238 238 F F F 23 460-479 144 144 144 250 250 250 T T T 24 480-499 141 141 141 245 245 245 F F F 25 500-519 142 142 142 247 247 247 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 140 140 140 244 244 244 T T T 28 560-579 137 137 137 239 239 239 T T T 29 580-599 141 141 141 245 245 245 T T T Max gen: 144, Max flush: 250 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 145 145 145 251 251 251 F F F 3 060-079 145 145 145 251 251 251 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 143 143 143 249 249 249 F F F 6 120-139 145 145 145 251 251 251 F F F 7 140-159 143 143 143 249 249 249 F F F 8 160-179 142 142 142 247 247 247 F F F 9 180-199 145 145 145 251 251 251 F F F 10 200-219 145 145 145 251 251 251 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 247 247 247 F F F 13 260-279 145 145 145 251 251 251 F F F 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 145 145 145 251 251 251 F F F 18 360-379 145 145 145 251 251 251 F F F 19 380-399 142 142 142 247 247 247 F F F 20 400-419 142 142 142 246 246 246 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 136 136 136 238 238 238 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 142 142 142 247 247 247 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 145 145 145 251 251 251 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 145, Max flush: 251 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.64426908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.690075048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.690681018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.690717896Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.690732216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.690746295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} {"msg":"Crucible 6e9b300a-2111-4f71-ac41-a47c28c7239e has session id: 3b1dadb1-e31f-4cc5-a613-f63df0770558","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.691203792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Upstairs opts: Upstairs UUID: 6e9b300a-2111-4f71-ac41-a47c28c7239e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.69124613Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Crucible stats registered with UUID: 6e9b300a-2111-4f71-ac41-a47c28c7239e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.69126432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721669294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721696373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} {"msg":"6e9b300a-2111-4f71-ac41-a47c28c7239e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721775299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721794408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721816907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721830777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721854755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721881354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721909403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721936491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.72196077Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.721982479Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_connection connected from Ok(127.0.0.1:37511)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722086094Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_connection connected from Ok(127.0.0.1:35208)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722121882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_connection connected from Ok(127.0.0.1:46966)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722303053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722840057Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722874365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722897214Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722963381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.722990149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.723022028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.723452007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.723521114Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.7235984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724059277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724100235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724131353Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724155662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724178971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.72420303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724224949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724250118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[0]R flush_numbers[0..12]: [245, 243, 251, 251, 247, 249, 251, 249, 247, 251, 251, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724277886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[0]R generation[0..12]: [141, 140, 145, 145, 142, 143, 145, 143, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724302195Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724328104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[1]R flush_numbers[0..12]: [245, 243, 251, 251, 247, 249, 251, 249, 247, 251, 251, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724355373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[1]R generation[0..12]: [141, 140, 145, 145, 142, 143, 145, 143, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724381491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.72440744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[2]R flush_numbers[0..12]: [245, 243, 251, 251, 247, 249, 251, 249, 247, 251, 251, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724436638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [141, 140, 145, 145, 142, 143, 145, 143, 142, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724474407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724505275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724533124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724561622Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724589441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724617349Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"":"downstairs","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724644978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAHDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724677877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"0","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724695116Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"1","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724708665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"client":"2","":"downstairs","session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724721924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724734354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"6e9b300a-2111-4f71-ac41-a47c28c7239e is now active with session: 3b1dadb1-e31f-4cc5-a613-f63df0770558","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724747243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724759242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766,"session_id":"3b1dadb1-e31f-4cc5-a613-f63df0770558"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:44.724771562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 260 len 28672 data: 13 12 13 14 14 12 10 02/30 Write block 196 len 20480 data: 13 12 11 11 12 03/30 Read block 522 len 40960 04/30 Flush 05/30 Flush 06/30 Write block 402 len 16384 data: 9 9 9 8 07/30 Read block 560 len 8192 08/30 Flush 09/30 Write block 447 len 12288 data: 13 12 5 10/30 Read block 299 len 12288 11/30 Write block 194 len 8192 data: 10 11 12/30 Write block 122 len 32768 data: 8 9 9 9 10 12 12 14 13/30 Write block 351 len 36864 data: 14 12 18 14 12 10 10 12 11 14/30 Read block 151 len 12288 15/30 Write block 549 len 16384 data: 10 8 7 5 16/30 Read block 21 len 32768 17/30 Read block 434 len 40960 18/30 Read block 281 len 16384 19/30 Write block 47 len 16384 data: 15 14 13 13 20/30 Read block 395 len 32768 21/30 Read block 548 len 32768 22/30 Write block 502 len 20480 data: 11 10 8 10 7 23/30 Write block 457 len 12288 data: 9 10 11 24/30 Read block 181 len 32768 25/30 Write block 211 len 8192 data: 10 9 26/30 Write block 164 len 8192 data: 13 11 27/30 Write block 389 len 12288 data: 8 10 11 28/30 Read block 128 len 16384 29/30 Read block 15 len 8192 30/30 Write block 201 len 8192 data: 14 12 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 3 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:44.743061533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 145 145 145 251 251 251 T T T 3 060-079 145 145 145 251 251 251 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 143 143 143 249 249 249 F F F 6 120-139 145 145 145 251 251 251 T T T 7 140-159 143 143 143 249 249 249 F F F 8 160-179 142 142 142 247 247 247 T T T 9 180-199 146 146 146 252 252 252 T T T 10 200-219 146 146 146 252 252 252 T T T 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 247 247 247 F F F 13 260-279 146 146 146 252 252 252 F F F 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 145 145 145 251 251 251 T T T 18 360-379 145 145 145 251 251 251 F F F 19 380-399 142 142 142 247 247 247 T T T 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 136 136 136 238 238 238 T T T 23 460-479 145 145 145 251 251 251 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 142 142 142 247 247 247 T T T 26 520-539 140 140 140 242 242 242 F F F 27 540-559 145 145 145 251 251 251 T T T 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 146, Max flush: 254 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 140 140 140 243 243 243 F F F 2 040-059 147 147 147 255 255 255 F F F 3 060-079 145 145 145 251 251 251 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 143 143 143 249 249 249 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 143 143 143 249 249 249 F F F 8 160-179 147 147 147 255 255 255 F F F 9 180-199 147 147 147 255 255 255 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 247 247 247 F F F 13 260-279 146 146 146 252 252 252 F F F 14 280-299 142 142 142 246 246 246 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 F F F 17 340-359 147 147 147 255 255 255 F F F 18 360-379 145 145 145 251 251 251 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 147 147 147 255 255 255 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 147 147 147 255 255 255 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 147 147 147 255 255 255 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 147, Max flush: 255 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.043768395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.089336984Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.089990892Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.09002918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.090043779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.090058389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} {"msg":"Crucible 0792a7d5-9e00-4975-883d-03b44b19870e has session id: c3a84a69-e2cf-4b30-a066-4215fe6d2ec0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.090534855Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Upstairs opts: Upstairs UUID: 0792a7d5-9e00-4975-883d-03b44b19870e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.090580883Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Crucible stats registered with UUID: 0792a7d5-9e00-4975-883d-03b44b19870e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.090605192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121409137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121452405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} {"msg":"0792a7d5-9e00-4975-883d-03b44b19870e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121514082Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121581129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121599728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121624677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121657075Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121683794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121709382Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121739151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121770619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36964)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12195775Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:65475)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.121995368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61083)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122013958Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122603909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122646176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122680355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122706493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.122739462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12277089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.123401719Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.123432238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.123532273Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12399104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124018279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124033388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124119104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124153192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124175871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12418948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12420337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[0]R flush_numbers[0..12]: [245, 243, 255, 251, 247, 249, 255, 249, 255, 255, 255, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124221029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[0]R generation[0..12]: [141, 140, 147, 145, 142, 143, 147, 143, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124235318Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124249657Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[1]R flush_numbers[0..12]: [245, 243, 255, 251, 247, 249, 255, 249, 255, 255, 255, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124263737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[1]R generation[0..12]: [141, 140, 147, 145, 142, 143, 147, 143, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124277416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124291515Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[2]R flush_numbers[0..12]: [245, 243, 255, 251, 247, 249, 255, 249, 255, 255, 255, 251]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124305475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[2]R generation[0..12]: [141, 140, 147, 145, 142, 143, 147, 143, 147, 147, 147, 145]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124319384Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124333464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124355712Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124380801Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.12440438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124418849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"":"downstairs","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124432608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124447728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"0","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124461577Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"1","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124475117Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"client":"2","":"downstairs","session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124488866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124501375Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0792a7d5-9e00-4975-883d-03b44b19870e is now active with session: c3a84a69-e2cf-4b30-a066-4215fe6d2ec0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124516964Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124529534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774,"session_id":"c3a84a69-e2cf-4b30-a066-4215fe6d2ec0"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:49.124542053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 530 len 16384 data: 12 11 10 8 02/30 Read block 300 len 36864 03/30 Write block 113 len 12288 data: 13 12 12 04/30 Read block 305 len 4096 05/30 Read block 585 len 12288 06/30 Write block 557 len 4096 data: 9 07/30 Write block 142 len 24576 data: 16 15 13 12 10 13 08/30 Read block 92 len 20480 09/30 Write block 230 len 40960 data: 14 13 14 13 13 13 14 14 12 12 10/30 Flush 11/30 Write block 190 len 36864 data: 7 9 8 10 11 12 14 13 12 12/30 Flush 13/30 Write block 505 len 16384 data: 11 8 6 7 14/30 Read block 41 len 28672 15/30 Read block 45 len 24576 16/30 Read block 343 len 8192 17/30 Write block 58 len 16384 data: 12 11 10 9 18/30 Write block 242 len 40960 data: 15 14 14 11 10 11 9 10 12 11 19/30 Write block 35 len 36864 data: 17 16 16 14 13 15 17 18 19 20/30 Write block 51 len 36864 data: 11 13 12 15 11 12 10 13 12 21/30 Read block 487 len 36864 22/30 Write block 156 len 16384 data: 13 13 13 14 23/30 Read block 435 len 12288 24/30 Write block 276 len 20480 data: 13 15 13 10 13 25/30 Write block 36 len 20480 data: 17 17 15 14 16 26/30 Read block 120 len 24576 27/30 Write block 0 len 24576 data: 4 5 7 7 8 11 28/30 Write block 338 len 20480 data: 10 10 8 8 9 29/30 Read block 278 len 20480 30/30 Read block 188 len 20480 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:49.14453557Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 T T T 1 020-039 140 140 140 243 243 243 T T T 2 040-059 147 147 147 255 255 255 T T T 3 060-079 145 145 145 251 251 251 T T T 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 148 148 148 256 256 256 T T T 8 160-179 147 147 147 255 255 255 F F F 9 180-199 148 148 148 257 257 257 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 148 148 148 256 256 256 F F F 12 240-259 142 142 142 247 247 247 T T T 13 260-279 146 146 146 252 252 252 T T T 14 280-299 142 142 142 246 246 246 T T T 15 300-319 142 142 142 246 246 246 F F F 16 320-339 142 142 142 247 247 247 T T T 17 340-359 147 147 147 255 255 255 T T T 18 360-379 145 145 145 251 251 251 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 147 147 147 255 255 255 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 147 147 147 255 255 255 T T T 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 148, Max flush: 257 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 258 258 258 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 149 149 149 258 258 258 F F F 8 160-179 147 147 147 255 255 255 F F F 9 180-199 148 148 148 257 257 257 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 148 148 148 256 256 256 F F F 12 240-259 149 149 149 258 258 258 F F F 13 260-279 149 149 149 258 258 258 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 149 149 149 258 258 258 F F F 17 340-359 149 149 149 258 258 258 F F F 18 360-379 145 145 145 251 251 251 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 147 147 147 255 255 255 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 149 149 149 258 258 258 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 145 145 145 251 251 251 F F F Max gen: 149, Max flush: 258 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.44896632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:53.492772016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.493179856Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.525588993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526214782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526246201Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.5262606Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526274959Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} {"msg":"Crucible 120400c3-9363-424f-9b3c-de82f5978f7b has session id: ca4c1a91-df0f-4c23-a30f-53860d7ea5d5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526734617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Upstairs opts: Upstairs UUID: 120400c3-9363-424f-9b3c-de82f5978f7b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526780914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Crucible stats registered with UUID: 120400c3-9363-424f-9b3c-de82f5978f7b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.526801813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557016468Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557043386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} {"msg":"120400c3-9363-424f-9b3c-de82f5978f7b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557114863Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557133252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557156581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.55717102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557184779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557197988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557211368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557224547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557238617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557252166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34765)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557417138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44358)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.557553051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57404)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.55758069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558175231Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558216029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558247947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558279565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558309654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558325063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.5588062Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.558961042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559128794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559430309Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559456768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559472187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559529974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559549893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559563672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.55960155Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559621759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[0]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 255, 258, 255, 257, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559645368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[0]R generation[0..12]: [149, 149, 149, 149, 142, 148, 147, 149, 147, 148, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559660027Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559674337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[1]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 255, 258, 255, 257, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559688456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[1]R generation[0..12]: [149, 149, 149, 149, 142, 148, 147, 149, 147, 148, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559702296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559716145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[2]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 255, 258, 255, 257, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559729894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[2]R generation[0..12]: [149, 149, 149, 149, 142, 148, 147, 149, 147, 148, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559743523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559757433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559770572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559783772Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559796861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.55981783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"":"downstairs","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559833079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559845518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"0","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559858998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"1","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559872267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"client":"2","":"downstairs","session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559885926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559898296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"120400c3-9363-424f-9b3c-de82f5978f7b is now active with session: ca4c1a91-df0f-4c23-a30f-53860d7ea5d5","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559911135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559923194Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782,"session_id":"ca4c1a91-df0f-4c23-a30f-53860d7ea5d5"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:53.559935394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 12288 data: 15 14 13 02/30 Write block 517 len 4096 data: 14 03/30 Read block 330 len 4096 04/30 Read block 123 len 28672 05/30 Write block 458 len 40960 data: 11 12 14 13 13 16 14 15 16 14 06/30 Flush 07/30 Write block 521 len 28672 data: 13 13 14 16 16 15 14 08/30 Flush 09/30 Write block 142 len 12288 data: 17 16 14 10/30 Read block 101 len 12288 11/30 Write block 444 len 32768 data: 20 17 17 14 13 6 4 3 12/30 Write block 586 len 16384 data: 9 10 10 11 13/30 Write block 504 len 32768 data: 9 12 9 7 8 10 9 11 14/30 Read block 455 len 8192 15/30 Write block 125 len 24576 data: 10 11 13 13 15 16 16/30 Read block 562 len 12288 17/30 Read block 72 len 36864 18/30 Write block 183 len 40960 data: 12 13 15 12 9 8 6 8 10 9 19/30 Read block 398 len 12288 20/30 Read block 516 len 28672 21/30 Write block 348 len 4096 data: 13 22/30 Read block 459 len 36864 23/30 Write block 177 len 4096 data: 8 24/30 Flush 25/30 Write block 259 len 8192 data: 8 14 26/30 Read block 386 len 40960 27/30 Write block 449 len 40960 data: 7 5 4 3 4 4 7 8 10 12 28/30 Read block 198 len 36864 29/30 Write block 153 len 32768 data: 5 8 10 14 14 14 15 13 30/30 Read block 85 len 4096 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:53.579781148Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 258 258 258 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 150 150 150 261 261 261 F F F 7 140-159 150 150 150 261 261 261 T T T 8 160-179 150 150 150 261 261 261 T T T 9 180-199 150 150 150 261 261 261 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 148 148 148 256 256 256 F F F 12 240-259 149 149 149 258 258 258 T T T 13 260-279 149 149 149 258 258 258 T T T 14 280-299 149 149 149 258 258 258 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 149 149 149 258 258 258 F F F 17 340-359 150 150 150 261 261 261 F F F 18 360-379 145 145 145 251 251 251 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 150 150 150 261 261 261 T T T 23 460-479 150 150 150 259 259 259 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 150 150 150 261 261 261 F F F Max gen: 150, Max flush: 261 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 258 258 258 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 150 150 150 261 261 261 F F F 7 140-159 151 151 151 262 262 262 F F F 8 160-179 151 151 151 262 262 262 F F F 9 180-199 150 150 150 261 261 261 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 148 148 148 256 256 256 F F F 12 240-259 151 151 151 262 262 262 F F F 13 260-279 151 151 151 262 262 262 F F F 14 280-299 149 149 149 258 258 258 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 149 149 149 258 258 258 F F F 17 340-359 150 150 150 261 261 261 F F F 18 360-379 145 145 145 251 251 251 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 151 151 151 262 262 262 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 141 141 141 245 245 245 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 150 150 150 260 260 260 F F F 27 540-559 148 148 148 256 256 256 F F F 28 560-579 145 145 145 251 251 251 F F F 29 580-599 150 150 150 261 261 261 F F F Max gen: 151, Max flush: 262 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.875714275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.921000348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.921832667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.921870346Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.921884755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.921899534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} {"msg":"Crucible f0dabdec-4740-415d-9698-15f5ab3c712d has session id: ec523563-0216-4ab8-ac83-7748105f14a0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.922293405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Upstairs opts: Upstairs UUID: f0dabdec-4740-415d-9698-15f5ab3c712d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.922338572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Crucible stats registered with UUID: f0dabdec-4740-415d-9698-15f5ab3c712d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.922363341Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.952911489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.952960487Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} {"msg":"f0dabdec-4740-415d-9698-15f5ab3c712d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.952989405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953012404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953039543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953057422Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953074961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95309407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953120119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953138178Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953155777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953176486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:51021)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953326129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62106)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953366337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54177)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.953395345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954215865Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954245494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95431048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95433019Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954392567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954408296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.9549352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954965528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.954980878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.9555352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955573088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955589388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955602187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955616186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955629756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955642165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955655514Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[0]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 261, 262, 262, 261, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955669784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[0]R generation[0..12]: [149, 149, 149, 149, 142, 148, 150, 151, 151, 150, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955683733Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955701632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[1]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 261, 262, 262, 261, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955715991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[1]R generation[0..12]: [149, 149, 149, 149, 142, 148, 150, 151, 151, 150, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955729901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95574383Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[2]R flush_numbers[0..12]: [258, 258, 258, 258, 247, 256, 261, 262, 262, 261, 255, 256]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955757729Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[2]R generation[0..12]: [149, 149, 149, 149, 142, 148, 150, 151, 151, 150, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955771659Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955785538Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955798627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-26T22:09:57.955824076Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955837715Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955851065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"":"downstairs","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955864604Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955876834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"0","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955890263Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"1","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955905572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"client":"2","":"downstairs","session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955919501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955932061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"f0dabdec-4740-415d-9698-15f5ab3c712d is now active with session: ec523563-0216-4ab8-ac83-7748105f14a0","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95594489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.95595701Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790,"session_id":"ec523563-0216-4ab8-ac83-7748105f14a0"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:09:57.955969399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 538 len 16384 data: 8 11 9 11 02/30 Write block 332 len 20480 data: 9 10 8 7 7 03/30 Write block 251 len 12288 data: 12 11 12 04/30 Read block 425 len 36864 05/30 Read block 385 len 28672 06/30 Write block 187 len 8192 data: 10 9 07/30 Read block 369 len 16384 08/30 Read block 412 len 20480 09/30 Flush 10/30 Write block 274 len 32768 data: 15 12 14 16 14 11 14 12 11/30 Read block 136 len 12288 12/30 Write block 290 len 40960 data: 14 10 10 8 10 13 13 15 10 12 13/30 Write block 369 len 40960 data: 10 13 11 10 11 10 12 14 11 11 14/30 Write block 126 len 36864 data: 12 14 14 16 17 17 16 19 16 15/30 Write block 234 len 24576 data: 14 14 15 15 13 13 16/30 Read block 315 len 28672 17/30 Read block 302 len 28672 18/30 Write block 280 len 8192 data: 15 13 19/30 Read block 504 len 40960 20/30 Flush 21/30 Write block 322 len 36864 data: 11 11 12 12 10 10 10 8 10 22/30 Write block 490 len 40960 data: 11 12 12 12 8 6 8 12 12 11 23/30 Read block 21 len 20480 24/30 Write block 563 len 16384 data: 14 12 14 15 25/30 Write block 161 len 16384 data: 13 12 9 14 26/30 Read block 18 len 16384 27/30 Read block 106 len 16384 28/30 Write block 222 len 24576 data: 9 8 10 12 15 15 29/30 Write block 355 len 24576 data: 13 11 11 13 12 11 30/30 Write block 8 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:09:57.974945616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 258 258 258 T T T 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 152 152 152 264 264 264 F F F 7 140-159 151 151 151 262 262 262 F F F 8 160-179 151 151 151 262 262 262 T T T 9 180-199 152 152 152 263 263 263 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 152 152 152 264 264 264 T T T 12 240-259 152 152 152 263 263 263 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 152 152 152 264 264 264 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 152 152 152 263 263 263 T T T 17 340-359 150 150 150 261 261 261 T T T 18 360-379 152 152 152 264 264 264 T T T 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 151 151 151 262 262 262 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 141 141 141 245 245 245 T T T 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 152 152 152 263 263 263 F F F 28 560-579 145 145 145 251 251 251 T T T 29 580-599 150 150 150 261 261 261 F F F Max gen: 152, Max flush: 264 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 152 152 152 264 264 264 F F F 7 140-159 151 151 151 262 262 262 F F F 8 160-179 153 153 153 265 265 265 F F F 9 180-199 152 152 152 263 263 263 F F F 10 200-219 147 147 147 255 255 255 F F F 11 220-239 153 153 153 265 265 265 F F F 12 240-259 152 152 152 263 263 263 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 152 152 152 264 264 264 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 153 153 153 265 265 265 F F F 17 340-359 153 153 153 265 265 265 F F F 18 360-379 153 153 153 265 265 265 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 151 151 151 262 262 262 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 153 153 153 265 265 265 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 152 152 152 263 263 263 F F F 28 560-579 153 153 153 265 265 265 F F F 29 580-599 150 150 150 261 261 261 F F F Max gen: 153, Max flush: 265 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.264713217Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.309557542Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.310149123Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.310186901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.31020152Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.31021632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} {"msg":"Crucible e572a3bc-31bc-40b8-83eb-dc93c87ffb38 has session id: 9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.310664637Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Upstairs opts: Upstairs UUID: e572a3bc-31bc-40b8-83eb-dc93c87ffb38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.310707786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Crucible stats registered with UUID: e572a3bc-31bc-40b8-83eb-dc93c87ffb38","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.310731714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341023355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341048774Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} {"msg":"e572a3bc-31bc-40b8-83eb-dc93c87ffb38 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341107831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.34112603Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341153978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341171717Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341192716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341213766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341246434Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341279672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.34132861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341371398Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58247)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341580348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40596)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341609886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51053)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.341633415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342153579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342178588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342192957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342206927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342220596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342234165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342802347Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.342941221Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343007737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343379849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343419887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343447696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343466005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343480564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343494133Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343506633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343519992Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[0]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 264, 262, 265, 263, 255, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.3435619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[0]R generation[0..12]: [153, 149, 149, 149, 142, 148, 152, 151, 153, 152, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343579189Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343593498Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[1]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 264, 262, 265, 263, 255, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343607308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[1]R generation[0..12]: [153, 149, 149, 149, 142, 148, 152, 151, 153, 152, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343624497Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343638996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[2]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 264, 262, 265, 263, 255, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343653006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[2]R generation[0..12]: [153, 149, 149, 149, 142, 148, 152, 151, 153, 152, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343666825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343680854Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343693864Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343707153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343720182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-11-26T22:10:02.343740491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"":"downstairs","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.34375413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.34376637Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"0","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343779689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"1","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343793129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"client":"2","":"downstairs","session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343806588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343818987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"e572a3bc-31bc-40b8-83eb-dc93c87ffb38 is now active with session: 9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343833517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343845636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"9c8cf991-2685-4fc7-9966-4a3fa4dc3dc6"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:02.343861515Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 157 len 32768 data: 15 15 16 14 14 13 10 15 02/30 Read block 60 len 16384 03/30 Write block 225 len 20480 data: 13 16 16 12 13 04/30 Write block 211 len 12288 data: 11 10 11 05/30 Read block 19 len 24576 06/30 Read block 187 len 20480 07/30 Flush 08/30 Write block 288 len 24576 data: 16 14 15 11 11 9 09/30 Write block 441 len 28672 data: 17 18 16 21 18 18 15 10/30 Read block 469 len 40960 11/30 Write block 148 len 16384 data: 10 9 11 11 12/30 Read block 102 len 4096 13/30 Flush 14/30 Write block 121 len 8192 data: 8 9 15/30 Write block 581 len 8192 data: 13 13 16/30 Read block 76 len 4096 17/30 Read block 394 len 32768 18/30 Write block 319 len 16384 data: 9 8 9 12 19/30 Read block 280 len 24576 20/30 Write block 222 len 20480 data: 10 9 11 14 17 21/30 Write block 495 len 4096 data: 7 22/30 Write block 321 len 32768 data: 10 13 12 13 13 11 11 11 23/30 Read block 20 len 16384 24/30 Write block 189 len 4096 data: 7 25/30 Flush 26/30 Write block 216 len 36864 data: 15 11 14 14 11 11 11 10 12 27/30 Read block 425 len 8192 28/30 Write block 552 len 40960 data: 6 6 7 8 8 10 12 12 12 12 29/30 Read block 194 len 8192 30/30 Read block 123 len 28672 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:02.362368435Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 154 154 154 266 266 266 F F F 9 180-199 154 154 154 268 268 268 F F F 10 200-219 154 154 154 266 266 266 T T T 11 220-239 154 154 154 268 268 268 T T T 12 240-259 152 152 152 263 263 263 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 154 154 154 267 267 267 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 154 154 154 268 268 268 F F F 17 340-359 153 153 153 265 265 265 F F F 18 360-379 153 153 153 265 265 265 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 154 154 154 268 268 268 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 152 152 152 263 263 263 T T T 28 560-579 153 153 153 265 265 265 T T T 29 580-599 154 154 154 268 268 268 F F F Max gen: 154, Max flush: 268 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 149 149 149 258 258 258 F F F 4 080-099 142 142 142 247 247 247 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 154 154 154 266 266 266 F F F 9 180-199 154 154 154 268 268 268 F F F 10 200-219 155 155 155 269 269 269 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 152 152 152 263 263 263 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 154 154 154 267 267 267 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 154 154 154 268 268 268 F F F 17 340-359 153 153 153 265 265 265 F F F 18 360-379 153 153 153 265 265 265 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 F F F 21 420-439 145 145 145 251 251 251 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 154 154 154 268 268 268 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 155 155 155 269 269 269 F F F 28 560-579 155 155 155 269 269 269 F F F 29 580-599 154 154 154 268 268 268 F F F Max gen: 155, Max flush: 269 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.626083607Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.670868845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.671509444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.671545342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.671564691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.671579521Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} {"msg":"Crucible f8cf88e5-dc87-4d0b-b167-ae28a1cb6818 has session id: c5845281-9342-4859-82f8-b7041f624f52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.672055237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Upstairs opts: Upstairs UUID: f8cf88e5-dc87-4d0b-b167-ae28a1cb6818, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.672103135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Crucible stats registered with UUID: f8cf88e5-dc87-4d0b-b167-ae28a1cb6818","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.672122854Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702443963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702471982Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} {"msg":"f8cf88e5-dc87-4d0b-b167-ae28a1cb6818 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702528619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702563527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702609785Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702646403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702682321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702717359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702751608Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702775107Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702804495Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702839244Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:36277)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702932489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:55559)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.702975007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:44060)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703006405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703433404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703474932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703506611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703536609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.70371452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.703742509Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704215826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704310441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.70434035Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.70473444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704759789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704779808Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704836295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704866164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704881493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704894352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704908092Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[0]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 268, 267, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704922141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[0]R generation[0..12]: [153, 149, 149, 149, 142, 148, 154, 154, 154, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.70493658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704951849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[1]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 268, 267, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704966029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[1]R generation[0..12]: [153, 149, 149, 149, 142, 148, 154, 154, 154, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704980018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.704994227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[2]R flush_numbers[0..12]: [265, 258, 258, 258, 247, 256, 268, 267, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705022036Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[2]R generation[0..12]: [153, 149, 149, 149, 142, 148, 154, 154, 154, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705040525Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705055144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705068364Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705081793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705094802Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705107852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"":"downstairs","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705129541Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.70514171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"0","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705154999Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"1","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705168259Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"client":"2","":"downstairs","session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705181638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705193958Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"f8cf88e5-dc87-4d0b-b167-ae28a1cb6818 is now active with session: c5845281-9342-4859-82f8-b7041f624f52","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705206897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705218967Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806,"session_id":"c5845281-9342-4859-82f8-b7041f624f52"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:06.705231226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 357 len 36864 data: 12 14 13 12 9 10 9 8 9 02/30 Write block 75 len 32768 data: 7 8 8 6 7 6 6 7 03/30 Write block 163 len 16384 data: 11 16 12 12 04/30 Read block 10 len 32768 05/30 Read block 413 len 20480 06/30 Read block 275 len 4096 07/30 Flush 08/30 Flush 09/30 Write block 289 len 4096 data: 15 10/30 Write block 188 len 20480 data: 10 8 9 11 10 11/30 Flush 12/30 Write block 337 len 4096 data: 8 13/30 Flush 14/30 Write block 181 len 36864 data: 12 12 13 14 16 13 11 11 9 15/30 Write block 248 len 20480 data: 10 11 13 13 12 16/30 Read block 61 len 32768 17/30 Read block 51 len 20480 18/30 Read block 298 len 24576 19/30 Write block 281 len 16384 data: 14 10 10 10 20/30 Write block 413 len 32768 data: 9 12 11 12 13 13 11 9 21/30 Read block 73 len 40960 22/30 Write block 364 len 36864 data: 9 10 10 9 9 11 14 12 11 23/30 Read block 363 len 28672 24/30 Read block 260 len 40960 25/30 Read block 406 len 36864 26/30 Read block 114 len 8192 27/30 Write block 483 len 20480 data: 15 14 10 10 9 28/30 Read block 315 len 32768 29/30 Read block 49 len 24576 30/30 Read block 260 len 8192 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:06.724482009Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 156 156 156 270 270 270 F F F 4 080-099 156 156 156 270 270 270 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 156 156 156 270 270 270 F F F 9 180-199 156 156 156 272 272 272 T T T 10 200-219 155 155 155 269 269 269 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 152 152 152 263 263 263 T T T 13 260-279 152 152 152 264 264 264 F F F 14 280-299 156 156 156 272 272 272 T T T 15 300-319 154 154 154 268 268 268 F F F 16 320-339 156 156 156 273 273 273 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 156 156 156 270 270 270 T T T 19 380-399 147 147 147 255 255 255 F F F 20 400-419 146 146 146 254 254 254 T T T 21 420-439 145 145 145 251 251 251 T T T 22 440-459 154 154 154 267 267 267 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 154 154 154 268 268 268 T T T 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 155 155 155 269 269 269 F F F 28 560-579 155 155 155 269 269 269 F F F 29 580-599 154 154 154 268 268 268 F F F Max gen: 156, Max flush: 273 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 149 149 149 258 258 258 F F F 2 040-059 149 149 149 258 258 258 F F F 3 060-079 156 156 156 270 270 270 F F F 4 080-099 156 156 156 270 270 270 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 156 156 156 270 270 270 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 155 155 155 269 269 269 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 157 157 157 274 274 274 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 157 157 157 274 274 274 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 156 156 156 273 273 273 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 157 157 157 274 274 274 F F F 21 420-439 157 157 157 274 274 274 F F F 22 440-459 154 154 154 267 267 267 F F F 23 460-479 150 150 150 259 259 259 F F F 24 480-499 157 157 157 274 274 274 F F F 25 500-519 150 150 150 261 261 261 F F F 26 520-539 152 152 152 263 263 263 F F F 27 540-559 155 155 155 269 269 269 F F F 28 560-579 155 155 155 269 269 269 F F F 29 580-599 154 154 154 268 268 268 F F F Max gen: 157, Max flush: 274 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.005797335Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.05124634Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.051899278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.051948156Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.051964015Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.051983024Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} {"msg":"Crucible 3d3ef502-77f4-4c81-8b21-f91f542136ea has session id: 351b2b24-351e-4487-ad2e-cb301f34c966","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.052396554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Upstairs opts: Upstairs UUID: 3d3ef502-77f4-4c81-8b21-f91f542136ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.052444492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Crucible stats registered with UUID: 3d3ef502-77f4-4c81-8b21-f91f542136ea","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.05247018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083419058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083445757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} {"msg":"3d3ef502-77f4-4c81-8b21-f91f542136ea active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083503014Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083521853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083544722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083560022Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083582381Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083648647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083677616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083704444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083732783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083778181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_connection connected from Ok(127.0.0.1:50235)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083808569Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_connection connected from Ok(127.0.0.1:52360)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083835628Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_connection connected from Ok(127.0.0.1:43634)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.083974691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084473367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084503115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084519114Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084533264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084554413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.084571362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085172122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.08521419Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085249288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085708736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085733835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085755974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.08582005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085849439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085864988Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085877857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085891777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[0]R flush_numbers[0..12]: [265, 258, 258, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085905656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[0]R generation[0..12]: [153, 149, 149, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085919795Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085933845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[1]R flush_numbers[0..12]: [265, 258, 258, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085947504Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[1]R generation[0..12]: [153, 149, 149, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085961084Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085974853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[2]R flush_numbers[0..12]: [265, 258, 258, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.085988822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[2]R generation[0..12]: [153, 149, 149, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.08603001Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086066088Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086087287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086121165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086149124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086170783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"":"downstairs","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086184582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086200002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"0","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.08622418Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"1","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086250739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"client":"2","":"downstairs","session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086274638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086287937Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"3d3ef502-77f4-4c81-8b21-f91f542136ea is now active with session: 351b2b24-351e-4487-ad2e-cb301f34c966","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086308756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086332205Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814,"session_id":"351b2b24-351e-4487-ad2e-cb301f34c966"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:11.086357044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 28672 data: 11 15 16 16 15 11 11 02/30 Read block 561 len 24576 03/30 Read block 191 len 24576 04/30 Write block 437 len 4096 data: 12 05/30 Flush 06/30 Write block 390 len 12288 data: 11 12 13 07/30 Read block 504 len 32768 08/30 Write block 507 len 32768 data: 8 9 11 10 12 11 11 11 09/30 Flush 10/30 Write block 52 len 24576 data: 14 13 16 12 13 11 11/30 Read block 518 len 20480 12/30 Read block 280 len 32768 13/30 Write block 576 len 32768 data: 14 14 12 12 11 14 14 13 14/30 Read block 542 len 28672 15/30 Read block 378 len 28672 16/30 Read block 544 len 8192 17/30 Write block 467 len 16384 data: 15 8 12 13 18/30 Flush 19/30 Write block 34 len 20480 data: 14 18 18 18 16 20/30 Flush 21/30 Write block 288 len 40960 data: 17 16 16 12 12 10 11 14 14 16 22/30 Write block 550 len 16384 data: 9 8 7 7 23/30 Write block 327 len 8192 data: 12 12 24/30 Flush 25/30 Write block 537 len 24576 data: 10 9 12 10 12 12 26/30 Write block 451 len 20480 data: 5 4 5 5 8 27/30 Write block 414 len 20480 data: 13 12 13 14 14 28/30 Write block 250 len 28672 data: 14 14 13 13 9 8 9 29/30 Read block 451 len 4096 30/30 Read block 218 len 40960 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:11.105499433Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 158 158 158 277 277 277 F F F 3 060-079 156 156 156 270 270 270 F F F 4 080-099 156 156 156 270 270 270 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 156 156 156 270 270 270 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 155 155 155 269 269 269 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 157 157 157 274 274 274 T T T 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 158 158 158 279 279 279 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 157 157 157 274 274 274 T T T 21 420-439 158 158 158 275 275 275 F F F 22 440-459 154 154 154 267 267 267 T T T 23 460-479 158 158 158 277 277 277 F F F 24 480-499 158 158 158 275 275 275 F F F 25 500-519 158 158 158 276 276 276 F F F 26 520-539 152 152 152 263 263 263 T T T 27 540-559 158 158 158 279 279 279 T T T 28 560-579 158 158 158 277 277 277 F F F 29 580-599 158 158 158 277 277 277 F F F Max gen: 158, Max flush: 279 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 158 158 158 277 277 277 F F F 3 060-079 156 156 156 270 270 270 F F F 4 080-099 156 156 156 270 270 270 F F F 5 100-119 148 148 148 256 256 256 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 F F F 8 160-179 156 156 156 270 270 270 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 155 155 155 269 269 269 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 159 159 159 280 280 280 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 158 158 158 279 279 279 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 159 159 159 280 280 280 F F F 21 420-439 158 158 158 275 275 275 F F F 22 440-459 159 159 159 280 280 280 F F F 23 460-479 158 158 158 277 277 277 F F F 24 480-499 158 158 158 275 275 275 F F F 25 500-519 158 158 158 276 276 276 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 159 159 159 280 280 280 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 158 158 158 277 277 277 F F F Max gen: 159, Max flush: 280 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.375577633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.421937893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.422540174Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.422575882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.422590051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.422605711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} {"msg":"Crucible 21b4a14c-adc4-4301-a90c-a5102daff2be has session id: a7012f07-c379-41c9-908f-969e0f5fca9d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.423091927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Upstairs opts: Upstairs UUID: 21b4a14c-adc4-4301-a90c-a5102daff2be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.423125275Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Crucible stats registered with UUID: 21b4a14c-adc4-4301-a90c-a5102daff2be","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.423143754Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.453987228Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454032475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} {"msg":"21b4a14c-adc4-4301-a90c-a5102daff2be active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454100252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45413597Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454166029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454198167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454232185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454256724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454287083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454312121Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45434535Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454380368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40109)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454495012Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55128)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454534661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37353)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.454569169Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455281764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455317252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455423667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455456305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455473144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.455498733Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45597547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456004938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456020568Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45657413Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456604039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456618838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456631207Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456645197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456658426Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456674135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456688005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[0]R flush_numbers[0..12]: [265, 278, 277, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456701894Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[0]R generation[0..12]: [153, 158, 158, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456715923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456729973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[1]R flush_numbers[0..12]: [265, 278, 277, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456743672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[1]R generation[0..12]: [153, 158, 158, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456757281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456771021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[2]R flush_numbers[0..12]: [265, 278, 277, 270, 270, 256, 268, 267, 270, 274, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45678486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[2]R generation[0..12]: [153, 158, 158, 156, 156, 148, 154, 154, 156, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456798829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456812798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456905644Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456931423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456949212Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456963271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"":"downstairs","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.456976661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.45698871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"0","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457002089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"1","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457015279Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"client":"2","":"downstairs","session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457028668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457040927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"21b4a14c-adc4-4301-a90c-a5102daff2be is now active with session: a7012f07-c379-41c9-908f-969e0f5fca9d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457053707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457065796Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822,"session_id":"a7012f07-c379-41c9-908f-969e0f5fca9d"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:15.457077876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 36864 data: 10 14 13 14 15 15 12 10 12 02/30 Read block 217 len 8192 03/30 Read block 153 len 20480 04/30 Read block 394 len 40960 05/30 Read block 93 len 20480 06/30 Write block 98 len 12288 data: 10 9 8 07/30 Read block 126 len 36864 08/30 Write block 546 len 16384 data: 15 14 13 11 09/30 Read block 126 len 16384 10/30 Read block 263 len 16384 11/30 Write block 204 len 8192 data: 13 11 12/30 Read block 11 len 40960 13/30 Read block 61 len 28672 14/30 Read block 474 len 36864 15/30 Write block 57 len 32768 data: 12 14 13 11 10 11 11 10 16/30 Read block 411 len 20480 17/30 Read block 173 len 20480 18/30 Read block 452 len 20480 19/30 Write block 414 len 4096 data: 15 20/30 Write block 461 len 36864 data: 14 14 17 15 16 17 16 9 13 21/30 Write block 106 len 4096 data: 13 22/30 Flush 23/30 Write block 586 len 20480 data: 10 11 11 12 12 24/30 Write block 586 len 12288 data: 11 12 12 25/30 Write block 97 len 32768 data: 11 11 10 9 11 12 13 13 26/30 Read block 13 len 16384 27/30 Read block 485 len 28672 28/30 Read block 73 len 12288 29/30 Write block 159 len 40960 data: 17 15 15 14 12 17 13 13 13 14 30/30 Write block 501 len 40960 data: 10 12 11 10 13 10 9 10 12 11 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:15.476561658Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 160 160 160 281 281 281 F F F 3 060-079 160 160 160 281 281 281 F F F 4 080-099 160 160 160 281 281 281 T T T 5 100-119 160 160 160 281 281 281 T T T 6 120-139 154 154 154 268 268 268 F F F 7 140-159 154 154 154 267 267 267 T T T 8 160-179 156 156 156 270 270 270 T T T 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 159 159 159 280 280 280 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 158 158 158 279 279 279 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 159 159 159 280 280 280 F F F 23 460-479 160 160 160 281 281 281 F F F 24 480-499 158 158 158 275 275 275 F F F 25 500-519 158 158 158 276 276 276 T T T 26 520-539 159 159 159 280 280 280 F F F 27 540-559 160 160 160 281 281 281 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 158 158 158 277 277 277 T T T Max gen: 160, Max flush: 281 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 160 160 160 281 281 281 F F F 3 060-079 160 160 160 281 281 281 F F F 4 080-099 161 161 161 282 282 282 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 154 154 154 268 268 268 F F F 7 140-159 161 161 161 282 282 282 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 155 155 155 269 269 269 F F F 12 240-259 159 159 159 280 280 280 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 158 158 158 279 279 279 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 159 159 159 280 280 280 F F F 23 460-479 160 160 160 281 281 281 F F F 24 480-499 158 158 158 275 275 275 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 160 160 160 281 281 281 F F F 28 560-579 158 158 158 277 277 277 F F F 29 580-599 161 161 161 282 282 282 F F F Max gen: 161, Max flush: 282 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.752933406Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.79780929Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.798454028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.798530874Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.798559143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.798575792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} {"msg":"Crucible 42822b23-f54c-4b7a-b663-acd9151bce6d has session id: 502f569b-59fb-4dac-966c-5d80635c78df","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.798987722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Upstairs opts: Upstairs UUID: 42822b23-f54c-4b7a-b663-acd9151bce6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.79903069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Crucible stats registered with UUID: 42822b23-f54c-4b7a-b663-acd9151bce6d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.799055908Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.8295248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829566738Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} {"msg":"42822b23-f54c-4b7a-b663-acd9151bce6d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829600177Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829619526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829666203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829706171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829745939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829786168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829845314Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829890262Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.82992811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829963759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_connection connected from Ok(127.0.0.1:52429)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.829995957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_connection connected from Ok(127.0.0.1:51967)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830190828Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43264)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830237075Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830880494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830924052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830966599Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.830999528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.831030716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.831061955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.831753991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.831919663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.831958971Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832415448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832445797Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832461396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832625208Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832654747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832670236Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832694425Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832709034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[0]R flush_numbers[0..12]: [265, 278, 281, 281, 282, 282, 268, 282, 282, 274, 281, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832772911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[0]R generation[0..12]: [153, 158, 160, 160, 161, 161, 154, 161, 161, 157, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.8327965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832811629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[1]R flush_numbers[0..12]: [265, 278, 281, 281, 282, 282, 268, 282, 282, 274, 281, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832825548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[1]R generation[0..12]: [153, 158, 160, 160, 161, 161, 154, 161, 161, 157, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832839157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832852987Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[2]R flush_numbers[0..12]: [265, 278, 281, 281, 282, 282, 268, 282, 282, 274, 281, 269]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832866656Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[2]R generation[0..12]: [153, 158, 160, 160, 161, 161, 154, 161, 161, 157, 160, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832880295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832894115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832906914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832920243Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832933553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832946632Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"":"downstairs","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:10:19.832960071Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.832977351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"0","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.83299322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"1","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833024528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"client":"2","":"downstairs","session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833039957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833052277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"42822b23-f54c-4b7a-b663-acd9151bce6d is now active with session: 502f569b-59fb-4dac-966c-5d80635c78df","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833065266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833077076Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830,"session_id":"502f569b-59fb-4dac-966c-5d80635c78df"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:19.833089305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 242 len 8192 data: 16 15 03/30 Write block 56 len 8192 data: 14 13 04/30 Write block 487 len 28672 data: 10 11 11 12 13 13 13 05/30 Read block 178 len 36864 06/30 Write block 328 len 4096 data: 13 07/30 Read block 519 len 20480 08/30 Read block 242 len 24576 09/30 Read block 430 len 40960 10/30 Read block 340 len 4096 11/30 Read block 74 len 40960 12/30 Flush 13/30 Write block 121 len 8192 data: 9 10 14/30 Read block 557 len 36864 15/30 Flush 16/30 Write block 570 len 16384 data: 16 15 16 18 17/30 Read block 178 len 24576 18/30 Read block 117 len 20480 19/30 Read block 41 len 40960 20/30 Write block 254 len 16384 data: 10 9 10 10 21/30 Flush 22/30 Write block 554 len 16384 data: 8 9 9 11 23/30 Read block 136 len 16384 24/30 Read block 189 len 24576 25/30 Read block 560 len 8192 26/30 Read block 390 len 12288 27/30 Write block 235 len 12288 data: 15 16 16 28/30 Read block 499 len 28672 29/30 Read block 256 len 36864 30/30 Write block 489 len 36864 data: 12 13 14 14 14 9 8 9 13 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:19.853866264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 160 160 160 281 281 281 F F F 4 080-099 161 161 161 282 282 282 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 162 162 162 285 285 285 F F F 7 140-159 161 161 161 282 282 282 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 155 155 155 269 269 269 T T T 12 240-259 162 162 162 286 286 286 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 162 162 162 284 284 284 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 159 159 159 280 280 280 F F F 23 460-479 160 160 160 281 281 281 F F F 24 480-499 162 162 162 284 284 284 T T T 25 500-519 161 161 161 282 282 282 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 160 160 160 281 281 281 T T T 28 560-579 162 162 162 286 286 286 F F F 29 580-599 161 161 161 282 282 282 F F F Max gen: 162, Max flush: 286 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 265 265 265 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 160 160 160 281 281 281 F F F 4 080-099 161 161 161 282 282 282 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 162 162 162 285 285 285 F F F 7 140-159 161 161 161 282 282 282 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 163 163 163 287 287 287 F F F 12 240-259 162 162 162 286 286 286 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 162 162 162 284 284 284 F F F 17 340-359 156 156 156 270 270 270 F F F 18 360-379 157 157 157 274 274 274 F F F 19 380-399 158 158 158 276 276 276 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 159 159 159 280 280 280 F F F 23 460-479 160 160 160 281 281 281 F F F 24 480-499 163 163 163 287 287 287 F F F 25 500-519 161 161 161 282 282 282 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 163 163 163 287 287 287 F F F 28 560-579 162 162 162 286 286 286 F F F 29 580-599 161 161 161 282 282 282 F F F Max gen: 163, Max flush: 287 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.111010979Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.156228715Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.156872164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.156909342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.156923471Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.156938981Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} {"msg":"Crucible 41773271-9d55-46e8-8275-d7d3004b78bb has session id: 15d48f80-1d16-4fb3-931a-613831d7e9df","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.157383229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Upstairs opts: Upstairs UUID: 41773271-9d55-46e8-8275-d7d3004b78bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.157428456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Crucible stats registered with UUID: 41773271-9d55-46e8-8275-d7d3004b78bb","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.157456715Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.187958135Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.187992623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} {"msg":"41773271-9d55-46e8-8275-d7d3004b78bb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.18807489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188099298Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188136767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188170055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188210183Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188244901Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.18827838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188306398Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188334517Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188359636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43570)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188387544Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_connection connected from Ok(127.0.0.1:62952)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.18846757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_connection connected from Ok(127.0.0.1:53856)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.188662741Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189252692Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.18928378Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189299759Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189314038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189327428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189350697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.189980826Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190018904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190051192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190542428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190583256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190613165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190673192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.19070287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190718029Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190730709Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190744888Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[0]R flush_numbers[0..12]: [265, 278, 284, 281, 282, 282, 285, 282, 282, 274, 281, 287]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190759547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[0]R generation[0..12]: [153, 158, 162, 160, 161, 161, 162, 161, 161, 157, 160, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190773807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190787886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[1]R flush_numbers[0..12]: [265, 278, 284, 281, 282, 282, 285, 282, 282, 274, 281, 287]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190802065Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[1]R generation[0..12]: [153, 158, 162, 160, 161, 161, 162, 161, 161, 157, 160, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190818804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190833124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[2]R flush_numbers[0..12]: [265, 278, 284, 281, 282, 282, 285, 282, 282, 274, 281, 287]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190846873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[2]R generation[0..12]: [153, 158, 162, 160, 161, 161, 162, 161, 161, 157, 160, 163]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190860493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190874392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190887441Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190900591Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190967427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190982916Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"":"downstairs","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.190996296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191008295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"0","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191021695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"1","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191034764Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"client":"2","":"downstairs","session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191047943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191060363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"41773271-9d55-46e8-8275-d7d3004b78bb is now active with session: 15d48f80-1d16-4fb3-931a-613831d7e9df","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191075442Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191087851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838,"session_id":"15d48f80-1d16-4fb3-931a-613831d7e9df"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:24.191100161Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 11 len 28672 data: 13 10 13 14 12 13 11 02/30 Write block 473 len 32768 data: 10 11 8 9 9 10 10 12 03/30 Write block 441 len 12288 data: 18 19 17 04/30 Flush 05/30 Write block 282 len 28672 data: 11 11 11 11 12 15 18 06/30 Read block 192 len 20480 07/30 Write block 228 len 32768 data: 13 14 15 14 15 14 15 16 08/30 Read block 530 len 4096 09/30 Read block 94 len 36864 10/30 Read block 295 len 24576 11/30 Write block 359 len 12288 data: 14 13 10 12/30 Write block 380 len 16384 data: 9 10 10 9 13/30 Read block 166 len 32768 14/30 Read block 573 len 8192 15/30 Flush 16/30 Write block 556 len 32768 data: 10 12 13 13 13 13 12 15 17/30 Read block 593 len 4096 18/30 Write block 150 len 40960 data: 12 12 7 6 9 11 15 16 16 18 19/30 Flush 20/30 Write block 382 len 16384 data: 11 10 9 11 21/30 Read block 157 len 8192 22/30 Write block 494 len 16384 data: 10 9 10 14 23/30 Write block 142 len 28672 data: 18 17 15 13 11 14 11 24/30 Write block 61 len 40960 data: 11 12 12 11 9 7 7 11 11 11 25/30 Write block 508 len 40960 data: 11 13 12 13 12 12 12 12 13 15 26/30 Flush 27/30 Write block 321 len 32768 data: 11 14 13 14 14 12 13 14 28/30 Write block 132 len 20480 data: 17 20 17 15 16 29/30 Write block 9 len 24576 data: 14 14 14 11 14 15 30/30 Read block 133 len 32768 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:24.210594572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 288 288 288 T T T 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 161 161 161 282 282 282 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 162 162 162 285 285 285 T T T 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 164 164 164 289 289 289 F F F 12 240-259 162 162 162 286 286 286 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 164 164 164 289 289 289 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 162 162 162 284 284 284 T T T 17 340-359 164 164 164 289 289 289 F F F 18 360-379 164 164 164 289 289 289 F F F 19 380-399 164 164 164 291 291 291 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 164 164 164 288 288 288 F F F 23 460-479 164 164 164 288 288 288 F F F 24 480-499 164 164 164 291 291 291 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 164 164 164 290 290 290 F F F 28 560-579 164 164 164 290 290 290 F F F 29 580-599 161 161 161 282 282 282 F F F Max gen: 164, Max flush: 291 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 161 161 161 282 282 282 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 165 165 165 292 292 292 F F F 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 157 157 157 274 274 274 F F F 10 200-219 160 160 160 281 281 281 F F F 11 220-239 164 164 164 289 289 289 F F F 12 240-259 162 162 162 286 286 286 F F F 13 260-279 152 152 152 264 264 264 F F F 14 280-299 164 164 164 289 289 289 F F F 15 300-319 154 154 154 268 268 268 F F F 16 320-339 165 165 165 292 292 292 F F F 17 340-359 164 164 164 289 289 289 F F F 18 360-379 164 164 164 289 289 289 F F F 19 380-399 164 164 164 291 291 291 F F F 20 400-419 160 160 160 281 281 281 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 164 164 164 288 288 288 F F F 23 460-479 164 164 164 288 288 288 F F F 24 480-499 164 164 164 291 291 291 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 159 159 159 280 280 280 F F F 27 540-559 164 164 164 290 290 290 F F F 28 560-579 164 164 164 290 290 290 F F F 29 580-599 161 161 161 282 282 282 F F F Max gen: 165, Max flush: 292 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.469047303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.513713667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.514424822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.51446305Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.514482299Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.514497168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} {"msg":"Crucible cefe91fe-7b72-46a6-9906-25b2d96d6004 has session id: e8824e71-1a41-4801-b567-70987095012d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.51487884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Upstairs opts: Upstairs UUID: cefe91fe-7b72-46a6-9906-25b2d96d6004, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.514924097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Crucible stats registered with UUID: cefe91fe-7b72-46a6-9906-25b2d96d6004","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.514949996Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546173761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546219228Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} {"msg":"cefe91fe-7b72-46a6-9906-25b2d96d6004 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546284395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546312704Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546336543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546351232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546364951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546377991Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.54639157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546408679Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546423018Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546482746Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51706)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546530423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54640)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546580251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41565)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.546830798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.547342003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.547371252Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.547387121Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.547400861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.54741409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.547427969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548096456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548122455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548137414Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548655949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548685227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548699866Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548767293Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548788122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548801861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548814181Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.54882744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[0]R flush_numbers[0..12]: [292, 278, 284, 291, 282, 282, 292, 291, 282, 274, 281, 289]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.54884143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[0]R generation[0..12]: [165, 158, 162, 164, 161, 161, 165, 164, 161, 157, 160, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548855209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548869338Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[1]R flush_numbers[0..12]: [292, 278, 284, 291, 282, 282, 292, 291, 282, 274, 281, 289]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548883168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[1]R generation[0..12]: [165, 158, 162, 164, 161, 161, 165, 164, 161, 157, 160, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548896867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548910756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[2]R flush_numbers[0..12]: [292, 278, 284, 291, 282, 282, 292, 291, 282, 274, 281, 289]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548924415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[2]R generation[0..12]: [165, 158, 162, 164, 161, 161, 165, 164, 161, 157, 160, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548941295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548955404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548968363Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548981403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.548994312Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549007162Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"":"downstairs","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549020231Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"Wait for a query_work_queue command to finish before sending IO session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.54904051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"0","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549054409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"1","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549075348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"client":"2","":"downstairs","session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549089947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549102227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"cefe91fe-7b72-46a6-9906-25b2d96d6004 is now active with session: e8824e71-1a41-4801-b567-70987095012d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549115536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549127596Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846,"session_id":"e8824e71-1a41-4801-b567-70987095012d"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:28.549139765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 593 len 28672 data: 11 10 11 8 7 5 2 03/30 Read block 351 len 8192 04/30 Write block 375 len 12288 data: 13 15 12 05/30 Read block 216 len 16384 06/30 Flush 07/30 Write block 310 len 36864 data: 14 15 15 14 14 14 11 8 9 08/30 Read block 24 len 32768 09/30 Read block 568 len 12288 10/30 Read block 24 len 4096 11/30 Read block 549 len 36864 12/30 Write block 527 len 12288 data: 15 12 12 13/30 Write block 258 len 8192 data: 10 9 14/30 Write block 400 len 12288 data: 9 8 10 15/30 Read block 350 len 20480 16/30 Read block 493 len 40960 17/30 Write block 221 len 16384 data: 12 12 11 13 18/30 Read block 400 len 24576 19/30 Write block 196 len 8192 data: 16 15 20/30 Read block 182 len 36864 21/30 Write block 208 len 16384 data: 7 8 8 12 22/30 Read block 465 len 4096 23/30 Write block 275 len 20480 data: 13 15 17 15 12 24/30 Flush 25/30 Write block 94 len 32768 data: 11 13 15 12 12 11 10 12 26/30 Write block 126 len 4096 data: 13 27/30 Read block 167 len 40960 28/30 Write block 347 len 20480 data: 9 14 14 13 15 29/30 Write block 478 len 12288 data: 11 11 13 30/30 Read block 223 len 8192 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 6 6 3 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:28.567356609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 161 161 161 282 282 282 T T T 5 100-119 161 161 161 282 282 282 T T T 6 120-139 165 165 165 292 292 292 T T T 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 166 166 166 295 295 295 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 166 166 166 295 295 295 F F F 13 260-279 166 166 166 295 295 295 F F F 14 280-299 164 164 164 289 289 289 F F F 15 300-319 166 166 166 295 295 295 F F F 16 320-339 165 165 165 292 292 292 F F F 17 340-359 164 164 164 289 289 289 T T T 18 360-379 166 166 166 294 294 294 F F F 19 380-399 164 164 164 291 291 291 F F F 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 164 164 164 288 288 288 F F F 23 460-479 164 164 164 288 288 288 T T T 24 480-499 164 164 164 291 291 291 T T T 25 500-519 164 164 164 291 291 291 F F F 26 520-539 166 166 166 295 295 295 F F F 27 540-559 164 164 164 290 290 290 F F F 28 560-579 164 164 164 290 290 290 F F F 29 580-599 166 166 166 294 294 294 F F F Max gen: 166, Max flush: 295 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 167 167 167 296 296 296 F F F 5 100-119 167 167 167 296 296 296 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 166 166 166 295 295 295 F F F 10 200-219 166 166 166 295 295 295 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 166 166 166 295 295 295 F F F 13 260-279 166 166 166 295 295 295 F F F 14 280-299 164 164 164 289 289 289 F F F 15 300-319 166 166 166 295 295 295 F F F 16 320-339 165 165 165 292 292 292 F F F 17 340-359 167 167 167 296 296 296 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 164 164 164 291 291 291 F F F 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 164 164 164 288 288 288 F F F 23 460-479 167 167 167 296 296 296 F F F 24 480-499 167 167 167 296 296 296 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 166 166 166 295 295 295 F F F 27 540-559 164 164 164 290 290 290 F F F 28 560-579 164 164 164 290 290 290 F F F 29 580-599 166 166 166 294 294 294 F F F Max gen: 167, Max flush: 296 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.841088119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.886057518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.886814891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.886852558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.886866948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.886881837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} {"msg":"Crucible bb68befd-ce8f-4594-a92e-b2d5a3921592 has session id: 486ec219-cefa-4cad-a985-3ebf08296a2f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.887477718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Upstairs opts: Upstairs UUID: bb68befd-ce8f-4594-a92e-b2d5a3921592, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.887520536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Crucible stats registered with UUID: bb68befd-ce8f-4594-a92e-b2d5a3921592","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.887536385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918211267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918240395Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} {"msg":"bb68befd-ce8f-4594-a92e-b2d5a3921592 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918314242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918332371Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.91835538Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918380308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918407837Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918436416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918504822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918579138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918629916Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918666654Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56320)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.918707782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51706)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919040376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37032)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919090173Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919624737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919653606Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919668485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919682124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.919706443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.91977404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920292924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920430518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920469066Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920886465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920920234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920936943Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.920979091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.92099975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921014109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921038448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921052237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[0]R flush_numbers[0..12]: [292, 278, 284, 291, 296, 296, 296, 291, 282, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921112904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[0]R generation[0..12]: [165, 158, 162, 164, 167, 167, 167, 164, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921133493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921148752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[1]R flush_numbers[0..12]: [292, 278, 284, 291, 296, 296, 296, 291, 282, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921162711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[1]R generation[0..12]: [165, 158, 162, 164, 167, 167, 167, 164, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921176511Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.92119034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[2]R flush_numbers[0..12]: [292, 278, 284, 291, 296, 296, 296, 291, 282, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921204149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[2]R generation[0..12]: [165, 158, 162, 164, 167, 167, 167, 164, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921217929Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921231868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921244878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921261267Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921274546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-11-26T22:10:32.921295685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"":"downstairs","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921309054Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921321044Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"0","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921334553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"1","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921348013Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"client":"2","":"downstairs","session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921361312Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921373571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"bb68befd-ce8f-4594-a92e-b2d5a3921592 is now active with session: 486ec219-cefa-4cad-a985-3ebf08296a2f","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921386391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.92139833Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854,"session_id":"486ec219-cefa-4cad-a985-3ebf08296a2f"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:32.921410639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 462 len 40960 data: 15 18 16 17 18 17 10 14 14 12 02/30 Write block 507 len 28672 data: 10 12 14 13 14 13 13 03/30 Write block 497 len 16384 data: 15 13 12 12 04/30 Read block 187 len 36864 05/30 Read block 517 len 20480 06/30 Write block 539 len 16384 data: 13 11 13 13 07/30 Write block 184 len 8192 data: 15 17 08/30 Flush 09/30 Write block 267 len 32768 data: 11 12 13 12 10 9 13 16 10/30 Flush 11/30 Write block 583 len 32768 data: 14 8 9 12 13 13 13 13 12/30 Read block 533 len 20480 13/30 Read block 235 len 28672 14/30 Write block 94 len 4096 data: 12 15/30 Write block 334 len 32768 data: 9 8 8 9 11 11 9 9 16/30 Write block 243 len 4096 data: 16 17/30 Write block 82 len 16384 data: 8 9 8 8 18/30 Write block 288 len 20480 data: 19 17 17 13 13 19/30 Flush 20/30 Write block 202 len 32768 data: 13 12 14 12 10 10 8 9 21/30 Read block 472 len 12288 22/30 Read block 238 len 24576 23/30 Read block 357 len 8192 24/30 Read block 280 len 40960 25/30 Write block 454 len 32768 data: 6 9 9 11 13 13 15 15 26/30 Write block 311 len 20480 data: 16 16 15 15 15 27/30 Write block 542 len 20480 data: 14 8 9 12 16 28/30 Read block 373 len 24576 29/30 Read block 549 len 4096 30/30 Write block 395 len 20480 data: 5 6 6 6 6 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:32.940217805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 167 167 167 296 296 296 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 166 166 166 295 295 295 T T T 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 166 166 166 295 295 295 T T T 16 320-339 168 168 168 299 299 299 F F F 17 340-359 168 168 168 299 299 299 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 164 164 164 291 291 291 T T T 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 164 164 164 288 288 288 T T T 23 460-479 168 168 168 297 297 297 T T T 24 480-499 168 168 168 297 297 297 F F F 25 500-519 168 168 168 297 297 297 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 168 168 168 297 297 297 T T T 28 560-579 164 164 164 290 290 290 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 168, Max flush: 299 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 162 162 162 284 284 284 F F F 3 060-079 164 164 164 291 291 291 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 167 167 167 296 296 296 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 164 164 164 291 291 291 F F F 8 160-179 161 161 161 282 282 282 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 169 169 169 300 300 300 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 169 169 169 300 300 300 F F F 16 320-339 168 168 168 299 299 299 F F F 17 340-359 168 168 168 299 299 299 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 169 169 169 300 300 300 F F F 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 169 169 169 300 300 300 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 168 168 168 297 297 297 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 164 164 164 290 290 290 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 169, Max flush: 300 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.224881607Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:37.269208857Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.269647635Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.3025085Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303092091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303128269Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303142129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303156588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} {"msg":"Crucible 92e92587-210a-447f-88b5-f02a02933205 has session id: a30cb08d-922a-4f26-aee6-0ff99ad905ce","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303640144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Upstairs opts: Upstairs UUID: 92e92587-210a-447f-88b5-f02a02933205, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303684522Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Crucible stats registered with UUID: 92e92587-210a-447f-88b5-f02a02933205","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.303701671Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33444354Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334474598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} {"msg":"92e92587-210a-447f-88b5-f02a02933205 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334543725Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334566874Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334594752Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334614531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33464537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334714976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334748134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334777813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334801202Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33482951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:44384)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334856409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:57621)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.334997482Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:46266)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335032581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335608882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335640101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33566476Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335687079Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335713367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.335743166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336292609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336345636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336379234Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336838762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33687918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336908978Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.336970515Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337005294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337021843Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337034962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337048601Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[0]R flush_numbers[0..12]: [292, 278, 284, 291, 299, 296, 296, 291, 282, 297, 300, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337062861Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[0]R generation[0..12]: [165, 158, 162, 164, 168, 167, 167, 164, 161, 168, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33707702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337091429Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[1]R flush_numbers[0..12]: [292, 278, 284, 291, 299, 296, 296, 291, 282, 297, 300, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337105329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[1]R generation[0..12]: [165, 158, 162, 164, 168, 167, 167, 164, 161, 168, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337119048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337133037Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[2]R flush_numbers[0..12]: [292, 278, 284, 291, 299, 296, 296, 291, 282, 297, 300, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337146977Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[2]R generation[0..12]: [165, 158, 162, 164, 168, 167, 167, 164, 161, 168, 169, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337160736Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337174735Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337187965Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337203434Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337225533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337238602Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"":"downstairs","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337251891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337264101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"0","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.33727742Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"1","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337290689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"client":"2","":"downstairs","session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337304439Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337316838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"92e92587-210a-447f-88b5-f02a02933205 is now active with session: a30cb08d-922a-4f26-aee6-0ff99ad905ce","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337329668Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337342567Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862,"session_id":"a30cb08d-922a-4f26-aee6-0ff99ad905ce"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:37.337354926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 334 len 36864 data: 10 9 9 10 12 12 10 10 10 03/30 Read block 267 len 12288 04/30 Write block 56 len 36864 data: 15 14 15 14 12 12 13 13 12 05/30 Read block 195 len 4096 06/30 Write block 518 len 4096 data: 16 07/30 Read block 157 len 4096 08/30 Write block 565 len 20480 data: 15 16 13 15 16 09/30 Flush 10/30 Write block 57 len 28672 data: 15 16 15 13 13 14 14 11/30 Read block 527 len 40960 12/30 Flush 13/30 Write block 333 len 36864 data: 11 11 10 10 11 13 13 11 11 14/30 Write block 155 len 32768 data: 12 16 17 17 19 16 16 15 15/30 Read block 344 len 32768 16/30 Read block 89 len 40960 17/30 Read block 391 len 4096 18/30 Flush 19/30 Write block 154 len 4096 data: 10 20/30 Read block 364 len 16384 21/30 Flush 22/30 Write block 214 len 8192 data: 11 13 23/30 Write block 442 len 16384 data: 20 18 22 19 24/30 Read block 579 len 8192 25/30 Read block 137 len 32768 26/30 Read block 242 len 24576 27/30 Write block 349 len 36864 data: 15 14 16 13 19 15 14 12 13 28/30 Read block 360 len 20480 29/30 Read block 76 len 12288 30/30 Read block 310 len 12288 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:37.356618049Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 170 170 170 303 303 303 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 167 167 167 296 296 296 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 170 170 170 305 305 305 F F F 8 160-179 170 170 170 304 304 304 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 169 169 169 300 300 300 T T T 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 169 169 169 300 300 300 F F F 16 320-339 170 170 170 304 304 304 F F F 17 340-359 170 170 170 304 304 304 T T T 18 360-379 166 166 166 294 294 294 F F F 19 380-399 169 169 169 300 300 300 F F F 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 169 169 169 300 300 300 T T T 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 170 170 170 302 302 302 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 170, Max flush: 305 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 F F F 2 040-059 170 170 170 303 303 303 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 167 167 167 296 296 296 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 170 170 170 305 305 305 F F F 8 160-179 170 170 170 304 304 304 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 171 171 171 306 306 306 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 169 169 169 300 300 300 F F F 16 320-339 170 170 170 304 304 304 F F F 17 340-359 171 171 171 306 306 306 F F F 18 360-379 166 166 166 294 294 294 F F F 19 380-399 169 169 169 300 300 300 F F F 20 400-419 166 166 166 295 295 295 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 171 171 171 306 306 306 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 170 170 170 302 302 302 F F F 29 580-599 168 168 168 299 299 299 F F F Max gen: 171, Max flush: 306 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.617330419Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.662198953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.662904578Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.662955115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.662976825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.663000953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} {"msg":"Crucible 23ad6fbb-4206-4d78-bcff-8cf4bb529528 has session id: 6ddfb30e-91d8-4bcc-b504-2e0b90958220","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.663444931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Upstairs opts: Upstairs UUID: 23ad6fbb-4206-4d78-bcff-8cf4bb529528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.663488069Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Crucible stats registered with UUID: 23ad6fbb-4206-4d78-bcff-8cf4bb529528","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.663512138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694477776Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694527773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} {"msg":"23ad6fbb-4206-4d78-bcff-8cf4bb529528 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694563551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694605039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694645097Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694676216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694702794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694730653Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694759462Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.69478853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694816639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694841388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_connection connected from Ok(127.0.0.1:53903)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694869546Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_connection connected from Ok(127.0.0.1:49061)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.694898165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_connection connected from Ok(127.0.0.1:43026)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695092835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695738214Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695771192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695787281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695801251Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.6958154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.695861137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.696388032Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.696474387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.696557103Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.696983342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697014961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.69703105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697102536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697121786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697135695Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697189172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697207721Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[0]R flush_numbers[0..12]: [292, 278, 303, 303, 299, 296, 296, 305, 304, 297, 306, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.6972258Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[0]R generation[0..12]: [165, 158, 170, 170, 168, 167, 167, 170, 170, 168, 171, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.69724049Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697254679Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[1]R flush_numbers[0..12]: [292, 278, 303, 303, 299, 296, 296, 305, 304, 297, 306, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697268718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[1]R generation[0..12]: [165, 158, 170, 170, 168, 167, 167, 170, 170, 168, 171, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697282308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697296087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[2]R flush_numbers[0..12]: [292, 278, 303, 303, 299, 296, 296, 305, 304, 297, 306, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697309786Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[2]R generation[0..12]: [165, 158, 170, 170, 168, 167, 167, 170, 170, 168, 171, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697323296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697337095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697349924Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697362784Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697375763Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:10:41.697388702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"":"downstairs","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697407721Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697421831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"0","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.69743512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"1","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697448179Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"client":"2","":"downstairs","session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697461629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697473968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"23ad6fbb-4206-4d78-bcff-8cf4bb529528 is now active with session: 6ddfb30e-91d8-4bcc-b504-2e0b90958220","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697486488Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697498427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870,"session_id":"6ddfb30e-91d8-4bcc-b504-2e0b90958220"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:41.697510706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 12288 data: 20 19 16 02/30 Read block 24 len 4096 03/30 Write block 333 len 8192 data: 12 12 04/30 Write block 374 len 24576 data: 11 14 16 13 12 9 05/30 Flush 06/30 Write block 202 len 20480 data: 14 13 15 13 11 07/30 Flush 08/30 Write block 377 len 28672 data: 14 13 10 10 11 12 11 09/30 Read block 497 len 16384 10/30 Read block 418 len 36864 11/30 Write block 395 len 16384 data: 6 7 7 7 12/30 Write block 395 len 24576 data: 7 8 8 8 7 10 13/30 Read block 584 len 12288 14/30 Read block 243 len 20480 15/30 Read block 102 len 4096 16/30 Read block 192 len 28672 17/30 Read block 108 len 28672 18/30 Flush 19/30 Write block 46 len 12288 data: 14 16 15 20/30 Read block 431 len 32768 21/30 Write block 111 len 24576 data: 18 18 14 13 13 12 22/30 Read block 68 len 16384 23/30 Flush 24/30 Write block 585 len 8192 data: 10 13 25/30 Read block 508 len 28672 26/30 Read block 206 len 8192 27/30 Read block 280 len 16384 28/30 Write block 159 len 8192 data: 20 17 29/30 Write block 22 len 20480 data: 11 13 13 8 9 30/30 Read block 407 len 12288 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:41.716235476Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 158 158 158 278 278 278 T T T 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 170 170 170 305 305 305 T T T 8 160-179 170 170 170 304 304 304 T T T 9 180-199 168 168 168 297 297 297 F F F 10 200-219 172 172 172 308 308 308 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 169 169 169 300 300 300 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 171 171 171 306 306 306 F F F 18 360-379 172 172 172 309 309 309 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 172 172 172 309 309 309 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 170 170 170 302 302 302 F F F 29 580-599 168 168 168 299 299 299 T T T Max gen: 172, Max flush: 310 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 173 173 173 311 311 311 F F F 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 173 173 173 311 311 311 F F F 8 160-179 173 173 173 311 311 311 F F F 9 180-199 168 168 168 297 297 297 F F F 10 200-219 172 172 172 308 308 308 F F F 11 220-239 166 166 166 295 295 295 F F F 12 240-259 168 168 168 299 299 299 F F F 13 260-279 168 168 168 298 298 298 F F F 14 280-299 168 168 168 299 299 299 F F F 15 300-319 169 169 169 300 300 300 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 171 171 171 306 306 306 F F F 18 360-379 172 172 172 309 309 309 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 172 172 172 309 309 309 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 168 168 168 297 297 297 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 170 170 170 302 302 302 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 173, Max flush: 311 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:45.985156342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.029905782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.030536731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.030573689Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.030587818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.030601947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} {"msg":"Crucible f4adf9fa-ed14-495c-a850-dc9fe25dcf23 has session id: 9282cea7-8c82-4d3b-9332-cee715e27188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.031084684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Upstairs opts: Upstairs UUID: f4adf9fa-ed14-495c-a850-dc9fe25dcf23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.031129811Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Crucible stats registered with UUID: f4adf9fa-ed14-495c-a850-dc9fe25dcf23","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.03115449Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061394713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061430101Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} {"msg":"f4adf9fa-ed14-495c-a850-dc9fe25dcf23 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.06146418Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061481329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061509208Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061530587Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061548256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061564705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061581304Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061597723Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061614762Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061631682Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_connection connected from Ok(127.0.0.1:60002)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061770955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_connection connected from Ok(127.0.0.1:43100)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061819352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_connection connected from Ok(127.0.0.1:50611)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.061959405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062595694Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062635532Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062792154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062831612Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062903369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.062944667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063323238Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063353787Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063634933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.06388942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063914129Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063929329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.063996045Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064016164Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064029813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064149038Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064168287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[0]R flush_numbers[0..12]: [292, 311, 310, 303, 299, 310, 296, 311, 311, 297, 308, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064186706Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[0]R generation[0..12]: [165, 173, 172, 170, 168, 172, 167, 173, 173, 168, 172, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064200885Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064215204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[1]R flush_numbers[0..12]: [292, 311, 310, 303, 299, 310, 296, 311, 311, 297, 308, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064233094Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[1]R generation[0..12]: [165, 173, 172, 170, 168, 172, 167, 173, 173, 168, 172, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064247283Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064261512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[2]R flush_numbers[0..12]: [292, 311, 310, 303, 299, 310, 296, 311, 311, 297, 308, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064275501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[2]R generation[0..12]: [165, 173, 172, 170, 168, 172, 167, 173, 173, 168, 172, 166]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064289211Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.06430301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.06431627Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064329499Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064342428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064355328Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"":"downstairs","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:10:46.064368327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064388716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"0","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064402095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"1","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064415265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"client":"2","":"downstairs","session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064428444Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064442633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"f4adf9fa-ed14-495c-a850-dc9fe25dcf23 is now active with session: 9282cea7-8c82-4d3b-9332-cee715e27188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064455593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064467562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878,"session_id":"9282cea7-8c82-4d3b-9332-cee715e27188"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:46.064479871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 4096 data: 13 02/30 Read block 459 len 28672 03/30 Flush 04/30 Flush 05/30 Write block 228 len 24576 data: 14 15 16 15 16 15 06/30 Write block 227 len 40960 data: 17 15 16 17 16 17 16 16 17 17 07/30 Read block 460 len 20480 08/30 Read block 272 len 36864 09/30 Write block 196 len 20480 data: 17 16 14 12 13 10/30 Write block 254 len 28672 data: 11 10 11 11 11 10 15 11/30 Write block 35 len 12288 data: 19 19 19 12/30 Read block 28 len 24576 13/30 Flush 14/30 Write block 215 len 8192 data: 14 16 15/30 Read block 576 len 36864 16/30 Write block 223 len 20480 data: 12 14 15 18 18 17/30 Read block 70 len 24576 18/30 Read block 73 len 32768 19/30 Write block 282 len 20480 data: 12 12 12 12 13 20/30 Read block 174 len 40960 21/30 Read block 66 len 16384 22/30 Read block 321 len 16384 23/30 Write block 360 len 32768 data: 14 11 11 10 10 11 11 10 24/30 Flush 25/30 Write block 522 len 36864 data: 14 15 17 17 16 16 13 13 13 26/30 Write block 365 len 4096 data: 12 27/30 Read block 123 len 4096 28/30 Write block 36 len 8192 data: 20 20 29/30 Write block 576 len 8192 data: 15 15 30/30 Write block 350 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:46.082528914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 174 174 174 314 314 314 T T T 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 173 173 173 311 311 311 F F F 8 160-179 173 173 173 311 311 311 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 174 174 174 315 315 315 F F F 12 240-259 174 174 174 314 314 314 F F F 13 260-279 174 174 174 314 314 314 F F F 14 280-299 174 174 174 315 315 315 F F F 15 300-319 174 174 174 312 312 312 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 171 171 171 306 306 306 T T T 18 360-379 174 174 174 315 315 315 T T T 19 380-399 172 172 172 309 309 309 F F F 20 400-419 172 172 172 309 309 309 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 168 168 168 297 297 297 T T T 27 540-559 169 169 169 300 300 300 F F F 28 560-579 170 170 170 302 302 302 T T T 29 580-599 173 173 173 311 311 311 F F F Max gen: 174, Max flush: 315 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 292 292 292 F F F 1 020-039 175 175 175 316 316 316 F F F 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 167 167 167 296 296 296 F F F 7 140-159 173 173 173 311 311 311 F F F 8 160-179 173 173 173 311 311 311 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 174 174 174 315 315 315 F F F 12 240-259 174 174 174 314 314 314 F F F 13 260-279 174 174 174 314 314 314 F F F 14 280-299 174 174 174 315 315 315 F F F 15 300-319 174 174 174 312 312 312 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 175 175 175 316 316 316 F F F 18 360-379 175 175 175 316 316 316 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 172 172 172 309 309 309 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 172 172 172 307 307 307 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 168 168 168 297 297 297 F F F 25 500-519 170 170 170 302 302 302 F F F 26 520-539 175 175 175 316 316 316 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 175 175 175 316 316 316 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 175, Max flush: 316 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.357051955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:50.401259631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.401664091Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.433906976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.434480048Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.434517016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.434530875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.434544985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} {"msg":"Crucible 0543aab7-3db5-47c6-bdbf-737f1df30fd9 has session id: de135b95-3e7b-4d27-959d-ab0c920537de","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.434978063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Upstairs opts: Upstairs UUID: 0543aab7-3db5-47c6-bdbf-737f1df30fd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.435022851Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Crucible stats registered with UUID: 0543aab7-3db5-47c6-bdbf-737f1df30fd9","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.43504074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46616386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466196818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} {"msg":"0543aab7-3db5-47c6-bdbf-737f1df30fd9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466269094Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466287744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466311063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466325492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466339281Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46635631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466402508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466419327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466439566Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466462775Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_connection connected from Ok(127.0.0.1:43455)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466489224Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_connection connected from Ok(127.0.0.1:33430)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466514262Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_connection connected from Ok(127.0.0.1:54738)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.466540321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467261526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467297914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467324753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467387199Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467415438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.467445277Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468038947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468086175Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468124063Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46860739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468632768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468647748Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468706615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468736533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468751592Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468764182Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468778021Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[0]R flush_numbers[0..12]: [292, 316, 310, 303, 299, 310, 296, 311, 311, 314, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46879454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[0]R generation[0..12]: [165, 175, 172, 170, 168, 172, 167, 173, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46880882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468823209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[1]R flush_numbers[0..12]: [292, 316, 310, 303, 299, 310, 296, 311, 311, 314, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468837138Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[1]R generation[0..12]: [165, 175, 172, 170, 168, 172, 167, 173, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468850838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468868047Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[2]R flush_numbers[0..12]: [292, 316, 310, 303, 299, 310, 296, 311, 311, 314, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468882526Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[2]R generation[0..12]: [165, 175, 172, 170, 168, 172, 167, 173, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468896605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468910825Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468923714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468936963Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468950003Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468963002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.468984231Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46899638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"0","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.46900961Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"1","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469022849Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"client":"2","":"downstairs","session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469036209Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469048498Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"0543aab7-3db5-47c6-bdbf-737f1df30fd9 is now active with session: de135b95-3e7b-4d27-959d-ab0c920537de","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469061327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469075456Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886,"session_id":"de135b95-3e7b-4d27-959d-ab0c920537de"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:50.469087976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 28672 data: 12 12 11 9 8 11 9 02/30 Write block 0 len 4096 data: 5 03/30 Read block 25 len 24576 04/30 Read block 355 len 28672 05/30 Read block 526 len 40960 06/30 Write block 271 len 16384 data: 11 10 14 17 07/30 Read block 440 len 8192 08/30 Read block 41 len 24576 09/30 Read block 527 len 24576 10/30 Read block 220 len 24576 11/30 Read block 331 len 20480 12/30 Read block 234 len 40960 13/30 Flush 14/30 Write block 234 len 12288 data: 17 18 18 15/30 Write block 499 len 28672 data: 13 13 11 13 12 11 14 16/30 Write block 446 len 16384 data: 20 17 14 8 17/30 Write block 401 len 32768 data: 9 11 10 10 9 9 12 11 18/30 Read block 89 len 36864 19/30 Write block 360 len 36864 data: 15 12 12 11 11 13 12 11 10 20/30 Write block 298 len 12288 data: 11 13 12 21/30 Write block 157 len 16384 data: 18 18 21 18 22/30 Write block 152 len 28672 data: 8 7 11 13 17 19 19 23/30 Read block 263 len 36864 24/30 Read block 390 len 40960 25/30 Read block 194 len 28672 26/30 Flush 27/30 Write block 259 len 28672 data: 11 16 13 14 15 15 13 28/30 Write block 510 len 16384 data: 14 15 14 14 29/30 Read block 84 len 20480 30/30 Write block 138 len 24576 data: 17 15 14 17 19 18 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:50.489016536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 317 317 317 F F F 1 020-039 175 175 175 316 316 316 F F F 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 167 167 167 296 296 296 T T T 7 140-159 176 176 176 318 318 318 T T T 8 160-179 176 176 176 318 318 318 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 174 174 174 314 314 314 T T T 13 260-279 176 176 176 317 317 317 T T T 14 280-299 176 176 176 318 318 318 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 176 176 176 317 317 317 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 176 176 176 318 318 318 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 176 176 176 318 318 318 T T T 26 520-539 175 175 175 316 316 316 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 175 175 175 316 316 316 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 176, Max flush: 318 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 317 317 317 F F F 1 020-039 175 175 175 316 316 316 F F F 2 040-059 172 172 172 310 310 310 F F F 3 060-079 170 170 170 303 303 303 F F F 4 080-099 168 168 168 299 299 299 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 176 176 176 318 318 318 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 176 176 176 318 318 318 F F F 12 240-259 177 177 177 319 319 319 F F F 13 260-279 177 177 177 319 319 319 F F F 14 280-299 176 176 176 318 318 318 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 176 176 176 317 317 317 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 176 176 176 318 318 318 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 177 177 177 319 319 319 F F F 26 520-539 175 175 175 316 316 316 F F F 27 540-559 169 169 169 300 300 300 F F F 28 560-579 175 175 175 316 316 316 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 177, Max flush: 319 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.792558089Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.838657322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839267492Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839304321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.83931819Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839335869Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} {"msg":"Crucible 5f1cd257-149c-41fc-8404-a6e427519e4d has session id: 5c2d8e1f-bd59-4755-a39e-3b65bdb5a596","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839788197Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Upstairs opts: Upstairs UUID: 5f1cd257-149c-41fc-8404-a6e427519e4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839826245Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Crucible stats registered with UUID: 5f1cd257-149c-41fc-8404-a6e427519e4d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.839849684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.87021823Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870263428Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} {"msg":"5f1cd257-149c-41fc-8404-a6e427519e4d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870329185Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870365593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870405611Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.87043503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870462798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870489227Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870514296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870539685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870565033Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870594352Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_connection connected from Ok(127.0.0.1:52196)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870621931Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_connection connected from Ok(127.0.0.1:46448)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870647519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_connection connected from Ok(127.0.0.1:54620)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.870799822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871334356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871362734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871502867Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871545005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871593593Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.871608542Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872044361Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872074919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872195743Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872619652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.87265955Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872692959Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872720387Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872747606Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872771235Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872792494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872816533Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[0]R flush_numbers[0..12]: [317, 316, 310, 303, 299, 310, 319, 319, 318, 314, 315, 318]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872842192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[0]R generation[0..12]: [176, 175, 172, 170, 168, 172, 177, 177, 176, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.87286716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872890899Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[1]R flush_numbers[0..12]: [317, 316, 310, 303, 299, 310, 319, 319, 318, 314, 315, 318]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872914808Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[1]R generation[0..12]: [176, 175, 172, 170, 168, 172, 177, 177, 176, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872940257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872967985Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[2]R flush_numbers[0..12]: [317, 316, 310, 303, 299, 310, 319, 319, 318, 314, 315, 318]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.872993154Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"[2]R generation[0..12]: [176, 175, 172, 170, 168, 172, 177, 177, 176, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873020783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873063581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873094559Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873123248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873149376Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873176325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"":"downstairs","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873203904Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:10:54.873228822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"0","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873256551Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"1","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.87327883Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"client":"2","":"downstairs","session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873293119Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873305679Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"5f1cd257-149c-41fc-8404-a6e427519e4d is now active with session: 5c2d8e1f-bd59-4755-a39e-3b65bdb5a596","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873318688Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873330948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894,"session_id":"5c2d8e1f-bd59-4755-a39e-3b65bdb5a596"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:54.873343527Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 20480 data: 15 16 16 17 14 02/30 Write block 228 len 32768 data: 16 17 18 17 18 17 18 19 03/30 Write block 11 len 4096 data: 15 04/30 Read block 446 len 4096 05/30 Read block 307 len 4096 06/30 Read block 563 len 32768 07/30 Read block 369 len 20480 08/30 Read block 124 len 32768 09/30 Write block 56 len 28672 data: 16 16 17 16 14 14 15 10/30 Read block 106 len 16384 11/30 Flush 12/30 Write block 55 len 16384 data: 13 17 17 18 13/30 Write block 98 len 4096 data: 13 14/30 Write block 32 len 8192 data: 10 10 15/30 Read block 547 len 8192 16/30 Write block 512 len 24576 data: 15 15 13 13 14 16 17/30 Read block 20 len 16384 18/30 Read block 247 len 20480 19/30 Write block 262 len 12288 data: 15 16 16 20/30 Write block 535 len 36864 data: 12 11 11 10 14 12 14 15 9 21/30 Write block 532 len 4096 data: 11 22/30 Write block 33 len 16384 data: 11 15 20 21 23/30 Write block 274 len 40960 data: 18 14 16 18 16 13 16 15 13 13 24/30 Write block 173 len 8192 data: 14 14 25/30 Flush 26/30 Write block 170 len 36864 data: 15 14 14 15 15 13 10 9 8 27/30 Read block 524 len 20480 28/30 Write block 267 len 28672 data: 12 13 14 13 12 11 15 29/30 Write block 79 len 20480 data: 8 7 7 9 10 30/30 Read block 346 len 4096 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:54.890724912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 320 320 320 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 178 178 178 321 321 321 F F F 3 060-079 178 178 178 320 320 320 T T T 4 080-099 178 178 178 321 321 321 T T T 5 100-119 172 172 172 310 310 310 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 178 178 178 321 321 321 T T T 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 178 178 178 320 320 320 F F F 12 240-259 177 177 177 319 319 319 F F F 13 260-279 178 178 178 321 321 321 T T T 14 280-299 178 178 178 321 321 321 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 178 178 178 320 320 320 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 176 176 176 318 318 318 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 178 178 178 321 321 321 F F F 28 560-579 175 175 175 316 316 316 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 178, Max flush: 321 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 320 320 320 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 178 178 178 321 321 321 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 179 179 179 322 322 322 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 178 178 178 320 320 320 F F F 12 240-259 177 177 177 319 319 319 F F F 13 260-279 179 179 179 322 322 322 F F F 14 280-299 178 178 178 321 321 321 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 178 178 178 320 320 320 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 172 172 172 309 309 309 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 176 176 176 318 318 318 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 178 178 178 321 321 321 F F F 28 560-579 175 175 175 316 316 316 F F F 29 580-599 173 173 173 311 311 311 F F F Max gen: 179, Max flush: 322 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.155548051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:59.200434373Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.200822925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.233272369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.233846491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.233883459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.233898368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.233912798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} {"msg":"Crucible 19ce18ea-3c65-4228-ab50-487458985b30 has session id: a7c360c5-7b28-40f0-9525-fd7d2d9dfd00","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.234351756Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Upstairs opts: Upstairs UUID: 19ce18ea-3c65-4228-ab50-487458985b30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.234394494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Crucible stats registered with UUID: 19ce18ea-3c65-4228-ab50-487458985b30","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.234411623Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.26497673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265035447Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} {"msg":"19ce18ea-3c65-4228-ab50-487458985b30 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265057976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265071875Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265092084Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265106484Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265120203Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265133732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265147332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265160911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265175Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.26518913Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_connection connected from Ok(127.0.0.1:48678)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265570171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_connection connected from Ok(127.0.0.1:40314)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265620408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_connection connected from Ok(127.0.0.1:65473)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.265652297Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.26620537Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266234448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266250337Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266273306Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266300345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266327584Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266877726Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.266953563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267045438Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267503716Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267533134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267548684Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.2676242Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267648619Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267662868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267675327Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267688727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[0]R flush_numbers[0..12]: [320, 321, 321, 322, 322, 310, 319, 319, 322, 314, 315, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267702676Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[0]R generation[0..12]: [178, 178, 178, 179, 179, 172, 177, 177, 179, 174, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267716835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267730935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[1]R flush_numbers[0..12]: [320, 321, 321, 322, 322, 310, 319, 319, 322, 314, 315, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267744914Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[1]R generation[0..12]: [178, 178, 178, 179, 179, 172, 177, 177, 179, 174, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267758673Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267772893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[2]R flush_numbers[0..12]: [320, 321, 321, 322, 322, 310, 319, 319, 322, 314, 315, 320]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267786792Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[2]R generation[0..12]: [178, 178, 178, 179, 179, 172, 177, 177, 179, 174, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267800591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267814691Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.26782999Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267843459Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267856638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267869728Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"":"downstairs","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:10:59.267883287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267902416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"0","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267915946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"1","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267929345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"client":"2","":"downstairs","session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267942804Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267955034Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"19ce18ea-3c65-4228-ab50-487458985b30 is now active with session: a7c360c5-7b28-40f0-9525-fd7d2d9dfd00","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267967793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267979882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902,"session_id":"a7c360c5-7b28-40f0-9525-fd7d2d9dfd00"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:10:59.267992122Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 12288 data: 11 12 15 02/30 Read block 60 len 40960 03/30 Write block 384 len 28672 data: 10 12 10 10 8 9 12 04/30 Read block 359 len 36864 05/30 Read block 375 len 36864 06/30 Read block 486 len 24576 07/30 Read block 588 len 4096 08/30 Write block 442 len 32768 data: 21 19 23 21 21 18 15 9 09/30 Read block 382 len 36864 10/30 Read block 518 len 40960 11/30 Write block 271 len 28672 data: 13 12 16 19 15 17 19 12/30 Write block 580 len 24576 data: 12 15 15 15 9 11 13/30 Write block 236 len 20480 data: 19 17 14 14 12 14/30 Read block 26 len 20480 15/30 Write block 259 len 24576 data: 12 17 14 16 17 17 16/30 Read block 332 len 16384 17/30 Flush 18/30 Write block 258 len 20480 data: 12 13 18 15 17 19/30 Read block 108 len 4096 20/30 Write block 570 len 8192 data: 17 16 21/30 Read block 568 len 28672 22/30 Read block 241 len 12288 23/30 Read block 548 len 8192 24/30 Read block 328 len 36864 25/30 Read block 420 len 36864 26/30 Read block 520 len 4096 27/30 Write block 242 len 40960 data: 17 17 15 12 11 12 11 12 15 15 28/30 Write block 273 len 40960 data: 17 20 16 18 20 17 14 17 16 14 29/30 Write block 42 len 8192 data: 19 20 30/30 Write block 539 len 40960 data: 15 13 15 16 10 10 13 17 15 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 1 4 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:10:59.287920872Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 320 320 320 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 178 178 178 321 321 321 T T T 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 179 179 179 322 322 322 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 180 180 180 323 323 323 F F F 12 240-259 180 180 180 323 323 323 T T T 13 260-279 180 180 180 323 323 323 T T T 14 280-299 180 180 180 323 323 323 T T T 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 178 178 178 320 320 320 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 180 180 180 323 323 323 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 178 178 178 321 321 321 T T T 27 540-559 178 178 178 321 321 321 T T T 28 560-579 175 175 175 316 316 316 T T T 29 580-599 180 180 180 323 323 323 F F F Max gen: 180, Max flush: 323 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 320 320 320 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 181 181 181 324 324 324 F F F 3 060-079 179 179 179 322 322 322 F F F 4 080-099 179 179 179 322 322 322 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 179 179 179 322 322 322 F F F 9 180-199 174 174 174 314 314 314 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 180 180 180 323 323 323 F F F 12 240-259 181 181 181 324 324 324 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 F F F 17 340-359 178 178 178 320 320 320 F F F 18 360-379 176 176 176 318 318 318 F F F 19 380-399 180 180 180 323 323 323 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 176 176 176 318 318 318 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 180 180 180 323 323 323 F F F Max gen: 181, Max flush: 324 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.601888342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:03.64546405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.64586403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.678244778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.67881582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.678851588Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.678866017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.678880677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} {"msg":"Crucible 523c81cf-2779-4924-b36b-769ca048f710 has session id: a102453d-d89c-420a-8b17-e450ece1f77c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.679276767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Upstairs opts: Upstairs UUID: 523c81cf-2779-4924-b36b-769ca048f710, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.679325005Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Crucible stats registered with UUID: 523c81cf-2779-4924-b36b-769ca048f710","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.679350874Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709643544Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709671713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} {"msg":"523c81cf-2779-4924-b36b-769ca048f710 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709744609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709767878Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709795906Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709814326Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709832455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709850064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709870393Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709900321Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.70993543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.709968548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64915)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710009266Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42832)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710045294Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58154)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710180348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710767789Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710807467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710840265Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710906882Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.710924471Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.71093911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.711422237Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.711452055Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.71154831Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.71195959Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.711989359Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712004548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712071995Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712096004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712110153Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712122572Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712135891Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[0]R flush_numbers[0..12]: [320, 321, 324, 322, 322, 310, 319, 319, 322, 314, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712149821Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[0]R generation[0..12]: [178, 178, 181, 179, 179, 172, 177, 177, 179, 174, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.71216365Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712177609Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[1]R flush_numbers[0..12]: [320, 321, 324, 322, 322, 310, 319, 319, 322, 314, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712191249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[1]R generation[0..12]: [178, 178, 181, 179, 179, 172, 177, 177, 179, 174, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712204818Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712218777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[2]R flush_numbers[0..12]: [320, 321, 324, 322, 322, 310, 319, 319, 322, 314, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712232617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[2]R generation[0..12]: [178, 178, 181, 179, 179, 172, 177, 177, 179, 174, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712246136Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712259935Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712272905Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712288004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712300973Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712313953Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"":"downstairs","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:03.712327232Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712347111Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"0","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.71236056Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"1","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.7123737Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"client":"2","":"downstairs","session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712386879Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712399118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"523c81cf-2779-4924-b36b-769ca048f710 is now active with session: a102453d-d89c-420a-8b17-e450ece1f77c","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712411918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712423757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910,"session_id":"a102453d-d89c-420a-8b17-e450ece1f77c"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:03.712435927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 24576 data: 8 8 9 12 11 12 02/30 Flush 03/30 Write block 59 len 36864 data: 17 15 15 16 15 13 10 8 8 04/30 Write block 496 len 4096 data: 11 05/30 Write block 94 len 36864 data: 13 14 16 13 14 12 11 13 13 06/30 Read block 268 len 16384 07/30 Read block 354 len 32768 08/30 Write block 256 len 8192 data: 12 12 09/30 Read block 137 len 36864 10/30 Write block 394 len 28672 data: 10 8 9 9 9 8 11 11/30 Write block 171 len 20480 data: 15 15 16 16 14 12/30 Write block 590 len 32768 data: 14 11 10 12 11 12 9 8 13/30 Read block 3 len 24576 14/30 Read block 490 len 12288 15/30 Read block 351 len 16384 16/30 Write block 184 len 8192 data: 16 18 17/30 Write block 181 len 20480 data: 13 13 14 17 19 18/30 Read block 467 len 40960 19/30 Read block 400 len 32768 20/30 Write block 365 len 24576 data: 14 13 12 11 12 15 21/30 Read block 258 len 32768 22/30 Read block 483 len 20480 23/30 Read block 360 len 40960 24/30 Write block 243 len 24576 data: 18 16 13 12 13 12 25/30 Read block 309 len 8192 26/30 Flush 27/30 Write block 320 len 12288 data: 9 12 15 28/30 Read block 373 len 24576 29/30 Read block 181 len 36864 30/30 Read block 92 len 24576 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:03.733047493Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 325 325 325 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 182 182 182 326 326 326 F F F 4 080-099 182 182 182 326 326 326 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 182 182 182 326 326 326 F F F 9 180-199 182 182 182 326 326 326 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 180 180 180 323 323 323 F F F 12 240-259 182 182 182 326 326 326 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 172 172 172 307 307 307 T T T 17 340-359 178 178 178 320 320 320 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 182 182 182 326 326 326 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 182 182 182 326 326 326 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 182, Max flush: 326 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 325 325 325 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 182 182 182 326 326 326 F F F 4 080-099 182 182 182 326 326 326 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 177 177 177 319 319 319 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 182 182 182 326 326 326 F F F 9 180-199 182 182 182 326 326 326 F F F 10 200-219 174 174 174 315 315 315 F F F 11 220-239 180 180 180 323 323 323 F F F 12 240-259 182 182 182 326 326 326 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 176 176 176 318 318 318 F F F 16 320-339 183 183 183 327 327 327 F F F 17 340-359 178 178 178 320 320 320 F F F 18 360-379 182 182 182 326 326 326 F F F 19 380-399 182 182 182 326 326 326 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 182 182 182 326 326 326 F F F 25 500-519 178 178 178 321 321 321 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 181 181 181 324 324 324 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 183, Max flush: 327 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:07.98137233Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:08.02509582Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.0255061Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.057893467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.05844975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058486548Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058500638Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058515287Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} {"msg":"Crucible dd51e62c-af8a-4ae0-bf9a-fa9c84ccf5e4 has session id: 9939cb6a-4775-4f0d-b70f-d068f8dc6f7d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058905848Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Upstairs opts: Upstairs UUID: dd51e62c-af8a-4ae0-bf9a-fa9c84ccf5e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058951446Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Crucible stats registered with UUID: dd51e62c-af8a-4ae0-bf9a-fa9c84ccf5e4","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.058978554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089443396Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089474485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} {"msg":"dd51e62c-af8a-4ae0-bf9a-fa9c84ccf5e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089498873Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089515183Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089542501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089560141Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.08957844Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089609838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089644486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089677405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089711523Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089744881Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60223)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.08977724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:32949)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089808708Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43930)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.089840647Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.090673206Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.090702494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.090717264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.090741732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.090769731Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.09079976Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091417389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091458807Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091489876Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091910755Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091936454Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.091950713Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092053288Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092076007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092089926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092106385Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092120475Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[0]R flush_numbers[0..12]: [325, 321, 326, 326, 326, 326, 319, 319, 326, 326, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092135144Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[0]R generation[0..12]: [182, 178, 182, 182, 182, 182, 177, 177, 182, 182, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092149143Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092163193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[1]R flush_numbers[0..12]: [325, 321, 326, 326, 326, 326, 319, 319, 326, 326, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092177172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[1]R generation[0..12]: [182, 178, 182, 182, 182, 182, 177, 177, 182, 182, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092190941Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092204711Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[2]R flush_numbers[0..12]: [325, 321, 326, 326, 326, 326, 319, 319, 326, 326, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.09221846Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[2]R generation[0..12]: [182, 178, 182, 182, 182, 182, 177, 177, 182, 182, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092232409Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092246329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092259358Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092272607Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092286356Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-11-26T22:11:08.092309835Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"":"downstairs","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092323605Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092341124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"0","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092366343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"1","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092383622Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"client":"2","":"downstairs","session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092397391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.09240993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"dd51e62c-af8a-4ae0-bf9a-fa9c84ccf5e4 is now active with session: 9939cb6a-4775-4f0d-b70f-d068f8dc6f7d","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.09242298Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092435109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918,"session_id":"9939cb6a-4775-4f0d-b70f-d068f8dc6f7d"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:08.092447739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 4096 data: 18 02/30 Write block 366 len 4096 data: 14 03/30 Read block 80 len 20480 04/30 Write block 207 len 40960 data: 11 9 10 9 13 11 12 12 15 17 05/30 Write block 367 len 8192 data: 13 12 06/30 Write block 248 len 32768 data: 13 13 16 16 14 14 12 11 07/30 Read block 410 len 12288 08/30 Read block 338 len 16384 09/30 Write block 328 len 20480 data: 15 9 11 9 10 10/30 Write block 340 len 36864 data: 13 13 12 10 9 12 10 10 16 11/30 Write block 181 len 32768 data: 14 14 15 18 20 14 12 12 12/30 Write block 129 len 24576 data: 17 18 18 18 21 18 13/30 Read block 421 len 28672 14/30 Read block 185 len 20480 15/30 Write block 503 len 20480 data: 13 12 15 11 11 16/30 Write block 70 len 36864 data: 12 9 9 10 9 8 9 9 7 17/30 Write block 189 len 28672 data: 10 10 12 11 11 12 13 18/30 Write block 216 len 4096 data: 18 19/30 Write block 359 len 40960 data: 15 16 13 13 12 12 15 15 14 13 20/30 Read block 437 len 8192 21/30 Write block 165 len 32768 data: 14 14 14 15 14 16 16 16 22/30 Read block 375 len 28672 23/30 Write block 313 len 36864 data: 16 16 16 12 9 10 10 10 13 24/30 Read block 346 len 28672 25/30 Read block 180 len 32768 26/30 Write block 506 len 8192 data: 12 12 27/30 Write block 1 len 4096 data: 6 28/30 Write block 222 len 24576 data: 13 13 15 16 19 19 29/30 Read block 87 len 16384 30/30 Write block 364 len 24576 data: 13 16 16 15 14 13 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:08.110801166Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 325 325 325 T T T 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 182 182 182 326 326 326 T T T 4 080-099 182 182 182 326 326 326 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 177 177 177 319 319 319 T T T 7 140-159 177 177 177 319 319 319 F F F 8 160-179 182 182 182 326 326 326 T T T 9 180-199 182 182 182 326 326 326 T T T 10 200-219 174 174 174 315 315 315 T T T 11 220-239 180 180 180 323 323 323 T T T 12 240-259 182 182 182 326 326 326 T T T 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 176 176 176 318 318 318 T T T 16 320-339 183 183 183 327 327 327 T T T 17 340-359 178 178 178 320 320 320 T T T 18 360-379 182 182 182 326 326 326 T T T 19 380-399 182 182 182 326 326 326 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 182 182 182 326 326 326 F F F 25 500-519 178 178 178 321 321 321 T T T 26 520-539 181 181 181 324 324 324 F F F 27 540-559 181 181 181 324 324 324 T T T 28 560-579 181 181 181 324 324 324 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 183, Max flush: 327 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 184 184 184 328 328 328 F F F 4 080-099 182 182 182 326 326 326 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 184 184 184 328 328 328 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 184 184 184 328 328 328 F F F 9 180-199 184 184 184 328 328 328 F F F 10 200-219 184 184 184 328 328 328 F F F 11 220-239 184 184 184 328 328 328 F F F 12 240-259 184 184 184 328 328 328 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 184 184 184 328 328 328 F F F 16 320-339 184 184 184 328 328 328 F F F 17 340-359 184 184 184 328 328 328 F F F 18 360-379 184 184 184 328 328 328 F F F 19 380-399 182 182 182 326 326 326 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 160 160 160 281 281 281 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 169 169 169 300 300 300 F F F 24 480-499 182 182 182 326 326 326 F F F 25 500-519 184 184 184 328 328 328 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 184 184 184 328 328 328 F F F 28 560-579 181 181 181 324 324 324 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 184, Max flush: 328 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.470993424Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:12.515052058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.515477877Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.548018137Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.548688734Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.548725662Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.548743591Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.548758051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} {"msg":"Crucible 37117516-f953-4f30-bcb3-3219d65ec568 has session id: 9542c908-7dba-4d4e-95fd-43c8f69d8b34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.54916727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Upstairs opts: Upstairs UUID: 37117516-f953-4f30-bcb3-3219d65ec568, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.549216028Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Crucible stats registered with UUID: 37117516-f953-4f30-bcb3-3219d65ec568","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.549235907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579493369Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579525248Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} {"msg":"37117516-f953-4f30-bcb3-3219d65ec568 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579555946Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579572615Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579599534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579616933Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579634303Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579651102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579674401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579703739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579742187Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579774345Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:39071)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579807834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:35793)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.579845372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_connection connected from Ok(127.0.0.1:56230)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.57988519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580598665Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580632513Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580771087Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580818134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580846463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.580861142Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581264942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581294391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581576707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581837944Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581866553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581882432Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.58192394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581942229Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.581955768Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582011186Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582030025Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[0]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 326, 328, 319, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582058783Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[0]R generation[0..12]: [184, 178, 182, 184, 182, 182, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582073343Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582087512Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[1]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 326, 328, 319, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582101401Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[1]R generation[0..12]: [184, 178, 182, 184, 182, 182, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.58211501Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.58212886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[2]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 326, 328, 319, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582142769Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[2]R generation[0..12]: [184, 178, 182, 184, 182, 182, 184, 177, 184, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582156708Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582173698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Max found gen is 185","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582187407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Generation requested: 186 >= found:185","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582200776Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582213766Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:12.582226685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"":"downstairs","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582244994Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582256934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"0","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582270093Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"1","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582283002Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"client":"2","":"downstairs","session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582296081Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582308341Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"37117516-f953-4f30-bcb3-3219d65ec568 is now active with session: 9542c908-7dba-4d4e-95fd-43c8f69d8b34","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.58232104Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.58233302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926,"session_id":"9542c908-7dba-4d4e-95fd-43c8f69d8b34"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:12.582345529Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 20480 data: 14 13 13 11 8 02/30 Read block 440 len 24576 03/30 Read block 457 len 12288 04/30 Read block 113 len 12288 05/30 Flush 06/30 Write block 403 len 8192 data: 11 11 07/30 Read block 489 len 40960 08/30 Write block 166 len 28672 data: 15 15 16 15 17 17 17 09/30 Write block 455 len 28672 data: 10 10 12 14 14 16 16 10/30 Read block 417 len 16384 11/30 Write block 113 len 28672 data: 15 14 14 13 11 11 11 12/30 Read block 523 len 40960 13/30 Write block 443 len 36864 data: 20 24 22 22 19 16 10 6 6 14/30 Read block 126 len 8192 15/30 Read block 156 len 36864 16/30 Write block 428 len 12288 data: 9 5 6 17/30 Write block 301 len 12288 data: 15 14 14 18/30 Read block 24 len 4096 19/30 Read block 366 len 16384 20/30 Read block 570 len 32768 21/30 Write block 219 len 36864 data: 15 12 13 14 14 16 17 20 20 22/30 Read block 571 len 40960 23/30 Read block 23 len 24576 24/30 Read block 429 len 16384 25/30 Write block 567 len 32768 data: 14 16 17 18 17 17 19 18 26/30 Read block 537 len 4096 27/30 Write block 398 len 32768 data: 10 9 12 10 12 12 12 10 28/30 Write block 352 len 12288 data: 15 20 16 29/30 Write block 546 len 28672 data: 19 16 15 12 10 9 8 30/30 Write block 445 len 28672 data: 23 23 20 17 11 7 7 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 2 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 4 10 Done 22 22 21 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:12.602441761Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 184 184 184 328 328 328 F F F 4 080-099 182 182 182 326 326 326 F F F 5 100-119 182 182 182 326 326 326 T T T 6 120-139 184 184 184 328 328 328 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 184 184 184 328 328 328 T T T 9 180-199 184 184 184 328 328 328 F F F 10 200-219 184 184 184 328 328 328 T T T 11 220-239 184 184 184 328 328 328 T T T 12 240-259 184 184 184 328 328 328 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 186 186 186 329 329 329 T T T 16 320-339 184 184 184 328 328 328 F F F 17 340-359 184 184 184 328 328 328 T T T 18 360-379 184 184 184 328 328 328 F F F 19 380-399 182 182 182 326 326 326 T T T 20 400-419 182 182 182 326 326 326 T T T 21 420-439 160 160 160 281 281 281 T T T 22 440-459 180 180 180 323 323 323 T T T 23 460-479 169 169 169 300 300 300 T T T 24 480-499 182 182 182 326 326 326 F F F 25 500-519 184 184 184 328 328 328 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 184 184 184 328 328 328 T T T 28 560-579 181 181 181 324 324 324 T T T 29 580-599 182 182 182 326 326 326 F F F Max gen: 186, Max flush: 329 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 178 178 178 321 321 321 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 184 184 184 328 328 328 F F F 4 080-099 182 182 182 326 326 326 F F F 5 100-119 187 187 187 330 330 330 F F F 6 120-139 184 184 184 328 328 328 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 187 187 187 330 330 330 F F F 9 180-199 184 184 184 328 328 328 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 187 187 187 330 330 330 F F F 12 240-259 184 184 184 328 328 328 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 181 181 181 324 324 324 F F F 15 300-319 187 187 187 330 330 330 F F F 16 320-339 184 184 184 328 328 328 F F F 17 340-359 187 187 187 330 330 330 F F F 18 360-379 184 184 184 328 328 328 F F F 19 380-399 187 187 187 330 330 330 F F F 20 400-419 187 187 187 330 330 330 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 187 187 187 330 330 330 F F F 24 480-499 182 182 182 326 326 326 F F F 25 500-519 184 184 184 328 328 328 F F F 26 520-539 181 181 181 324 324 324 F F F 27 540-559 187 187 187 330 330 330 F F F 28 560-579 187 187 187 330 330 330 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 187, Max flush: 330 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:16.946622225Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:16.992751827Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:16.993138698Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.025367664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.025930076Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.025969934Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.025984393Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.026001853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} {"msg":"Crucible e95c07c0-343a-4baa-989d-acf52ad2197e has session id: 63f272a7-560f-4238-985a-db56d28d4b86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.026423332Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Upstairs opts: Upstairs UUID: e95c07c0-343a-4baa-989d-acf52ad2197e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.02646667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Crucible stats registered with UUID: e95c07c0-343a-4baa-989d-acf52ad2197e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.026485389Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.056973319Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057007308Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} {"msg":"e95c07c0-343a-4baa-989d-acf52ad2197e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057032157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057048416Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057076085Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057093974Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057111113Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057136102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.05717349Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057204938Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057231467Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057257696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_connection connected from Ok(127.0.0.1:60793)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057283884Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_connection connected from Ok(127.0.0.1:46964)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057314853Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_connection connected from Ok(127.0.0.1:48502)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.057485744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058095554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058127553Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058143292Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058157651Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.05817336Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058200399Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058838718Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058869176Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.058888505Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059418139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059448198Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059463947Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059543333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059565282Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059579102Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059591291Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.05960455Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[0]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 330, 328, 319, 330, 328, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.05961868Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[0]R generation[0..12]: [184, 178, 182, 184, 182, 187, 184, 177, 187, 184, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059632989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059647208Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[1]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 330, 328, 319, 330, 328, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059660998Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[1]R generation[0..12]: [184, 178, 182, 184, 182, 187, 184, 177, 187, 184, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059674907Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059688926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[2]R flush_numbers[0..12]: [328, 321, 326, 328, 326, 330, 328, 319, 330, 328, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059705975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[2]R generation[0..12]: [184, 178, 182, 184, 182, 187, 184, 177, 187, 184, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059720124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059734064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059747083Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059760362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059773282Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"All extents match","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2024-11-26T22:11:17.059835969Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"":"downstairs","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059876397Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059902486Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"0","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059929204Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"1","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059955053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"client":"2","":"downstairs","session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.059977842Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.06001483Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "} {"msg":"e95c07c0-343a-4baa-989d-acf52ad2197e is now active with session: 63f272a7-560f-4238-985a-db56d28d4b86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.060048408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.060067547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934,"session_id":"63f272a7-560f-4238-985a-db56d28d4b86"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:17.060083617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 25 len 8192 data: 9 10 02/30 Read block 147 len 16384 03/30 Flush 04/30 Write block 408 len 36864 data: 12 14 10 10 10 11 16 14 15 05/30 Flush 06/30 Write block 486 len 16384 data: 12 11 12 13 07/30 Write block 62 len 36864 data: 17 16 14 11 9 9 12 12 13 08/30 Write block 161 len 36864 data: 17 16 13 18 15 16 16 17 16 09/30 Read block 500 len 28672 10/30 Flush 11/30 Write block 363 len 12288 data: 13 14 17 12/30 Read block 107 len 4096 13/30 Read block 530 len 4096 14/30 Write block 395 len 40960 data: 9 10 10 11 10 13 11 13 13 13 15/30 Flush 16/30 Flush 17/30 Write block 493 len 8192 data: 15 11 18/30 Write block 35 len 28672 data: 21 22 21 17 15 17 18 19/30 Read block 251 len 12288 20/30 Write block 292 len 32768 data: 14 12 13 16 15 17 12 14 21/30 Write block 22 len 36864 data: 12 14 14 10 11 8 7 6 8 22/30 Flush 23/30 Write block 523 len 28672 data: 16 18 18 17 17 14 14 24/30 Write block 324 len 24576 data: 15 15 13 14 16 10 25/30 Write block 60 len 4096 data: 16 26/30 Write block 186 len 16384 data: 15 13 13 11 27/30 Write block 95 len 12288 data: 15 17 14 28/30 Read block 547 len 24576 29/30 Read block 192 len 4096 30/30 Read block 405 len 40960 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:17.078486162Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 188 188 188 333 333 333 T T T 4 080-099 182 182 182 326 326 326 T T T 5 100-119 187 187 187 330 330 330 F F F 6 120-139 184 184 184 328 328 328 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 184 184 184 328 328 328 T T T 10 200-219 187 187 187 330 330 330 F F F 11 220-239 187 187 187 330 330 330 F F F 12 240-259 184 184 184 328 328 328 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 188 188 188 336 336 336 F F F 15 300-319 187 187 187 330 330 330 F F F 16 320-339 184 184 184 328 328 328 T T T 17 340-359 187 187 187 330 330 330 F F F 18 360-379 188 188 188 334 334 334 F F F 19 380-399 188 188 188 334 334 334 F F F 20 400-419 188 188 188 334 334 334 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 187 187 187 330 330 330 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 184 184 184 328 328 328 F F F 26 520-539 181 181 181 324 324 324 T T T 27 540-559 187 187 187 330 330 330 F F F 28 560-579 187 187 187 330 330 330 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 188, Max flush: 336 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 189 189 189 337 337 337 F F F 5 100-119 187 187 187 330 330 330 F F F 6 120-139 184 184 184 328 328 328 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 189 189 189 337 337 337 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 187 187 187 330 330 330 F F F 12 240-259 184 184 184 328 328 328 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 188 188 188 336 336 336 F F F 15 300-319 187 187 187 330 330 330 F F F 16 320-339 189 189 189 337 337 337 F F F 17 340-359 187 187 187 330 330 330 F F F 18 360-379 188 188 188 334 334 334 F F F 19 380-399 188 188 188 334 334 334 F F F 20 400-419 188 188 188 334 334 334 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 187 187 187 330 330 330 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 184 184 184 328 328 328 F F F 26 520-539 189 189 189 337 337 337 F F F 27 540-559 187 187 187 330 330 330 F F F 28 560-579 187 187 187 330 330 330 F F F 29 580-599 182 182 182 326 326 326 F F F Max gen: 189, Max flush: 337 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.385183871Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.430164629Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.430785408Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.430824006Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.430838226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.430856565Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} {"msg":"Crucible 415d688e-bf86-434c-9a45-1ae7cebd1515 has session id: 6619023b-359a-41a9-9dc5-d47ea352057e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.431313652Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Upstairs opts: Upstairs UUID: 415d688e-bf86-434c-9a45-1ae7cebd1515, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.43135824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Crucible stats registered with UUID: 415d688e-bf86-434c-9a45-1ae7cebd1515","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.431383939Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461593503Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461622112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} {"msg":"415d688e-bf86-434c-9a45-1ae7cebd1515 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461680139Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461721747Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461820392Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.46186663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461906558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.461943446Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462117508Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462151816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462180214Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462214563Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47868)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462244451Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46313)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.46227151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49100)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462290289Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462800064Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462839952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462870461Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462940597Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462969016Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.462985175Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.463439333Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.463617494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.4636997Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464030794Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464055812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464074041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464115039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464132778Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464146078Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464183476Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464200215Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[0]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 330, 328, 319, 333, 337, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464220624Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[0]R generation[0..12]: [184, 188, 188, 189, 189, 187, 184, 177, 188, 189, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464234773Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464248793Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[1]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 330, 328, 319, 333, 337, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464262482Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[1]R generation[0..12]: [184, 188, 188, 189, 189, 187, 184, 177, 188, 189, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464276202Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464290191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[2]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 330, 328, 319, 333, 337, 330, 330]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.46430397Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[2]R generation[0..12]: [184, 188, 188, 189, 189, 187, 184, 177, 188, 189, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.46431758Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464333829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464346968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464360167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464373007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464385926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"":"downstairs","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:21.464398805Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464418724Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"0","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464431754Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"1","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464444633Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"client":"2","":"downstairs","session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464457722Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464469952Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"415d688e-bf86-434c-9a45-1ae7cebd1515 is now active with session: 6619023b-359a-41a9-9dc5-d47ea352057e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464482631Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.464494531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942,"session_id":"6619023b-359a-41a9-9dc5-d47ea352057e"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:21.4645149Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 20480 data: 14 14 14 14 13 02/30 Read block 380 len 24576 03/30 Write block 221 len 36864 data: 14 15 15 17 18 21 21 17 18 04/30 Write block 320 len 36864 data: 11 14 16 14 16 16 14 15 17 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 578 len 32768 data: 13 13 13 16 16 16 10 12 09/30 Flush 10/30 Write block 308 len 24576 data: 16 12 15 17 17 17 11/30 Write block 538 len 12288 data: 11 16 14 12/30 Flush 13/30 Write block 119 len 40960 data: 12 11 10 11 10 10 11 14 15 15 14/30 Read block 77 len 28672 15/30 Read block 74 len 32768 16/30 Read block 221 len 36864 17/30 Read block 210 len 28672 18/30 Read block 314 len 8192 19/30 Read block 293 len 40960 20/30 Read block 382 len 24576 21/30 Write block 245 len 32768 data: 14 13 14 14 14 17 17 15 22/30 Read block 19 len 36864 23/30 Write block 354 len 36864 data: 17 15 13 14 15 16 17 14 14 24/30 Read block 228 len 8192 25/30 Read block 184 len 36864 26/30 Write block 237 len 40960 data: 18 15 15 13 15 18 19 17 15 14 27/30 Read block 30 len 16384 28/30 Write block 515 len 16384 data: 14 15 17 17 29/30 Read block 497 len 16384 30/30 Read block 502 len 40960 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 10 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:21.486436052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 189 189 189 337 337 337 F F F 5 100-119 187 187 187 330 330 330 T T T 6 120-139 184 184 184 328 328 328 T T T 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 189 189 189 337 337 337 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 190 190 190 338 338 338 T T T 12 240-259 184 184 184 328 328 328 T T T 13 260-279 181 181 181 324 324 324 F F F 14 280-299 188 188 188 336 336 336 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 190 190 190 338 338 338 F F F 17 340-359 187 187 187 330 330 330 T T T 18 360-379 188 188 188 334 334 334 T T T 19 380-399 188 188 188 334 334 334 F F F 20 400-419 188 188 188 334 334 334 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 187 187 187 330 330 330 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 184 184 184 328 328 328 T T T 26 520-539 190 190 190 342 342 342 F F F 27 540-559 190 190 190 342 342 342 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 190 190 190 341 341 341 F F F Max gen: 190, Max flush: 342 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 189 189 189 337 337 337 F F F 5 100-119 191 191 191 343 343 343 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 189 189 189 337 337 337 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 191 191 191 343 343 343 F F F 12 240-259 191 191 191 343 343 343 F F F 13 260-279 181 181 181 324 324 324 F F F 14 280-299 188 188 188 336 336 336 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 190 190 190 338 338 338 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 191 191 191 343 343 343 F F F 19 380-399 188 188 188 334 334 334 F F F 20 400-419 188 188 188 334 334 334 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 187 187 187 330 330 330 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 F F F 26 520-539 190 190 190 342 342 342 F F F 27 540-559 190 190 190 342 342 342 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 190 190 190 341 341 341 F F F Max gen: 191, Max flush: 343 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.76915617Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.815983616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.816760828Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.816799696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.816814096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.816828415Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} {"msg":"Crucible d4ae14ae-f80e-4156-9cde-e1a7eb0cff3e has session id: fa47f522-a9b4-418f-8f38-669129cf0b5b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.81734798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Upstairs opts: Upstairs UUID: d4ae14ae-f80e-4156-9cde-e1a7eb0cff3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.817394767Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Crucible stats registered with UUID: d4ae14ae-f80e-4156-9cde-e1a7eb0cff3e","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.817415386Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847565664Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847597362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} {"msg":"d4ae14ae-f80e-4156-9cde-e1a7eb0cff3e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847626661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.84764347Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847671529Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847690017Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847707007Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847731915Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847765074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847797902Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847832041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.847856799Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52730)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848075549Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36481)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848117847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57922)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848147105Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.84866502Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848806443Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848831191Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848846491Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848899218Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.848941216Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.849380824Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.849422822Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.849450951Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.849981945Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850046702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.85007494Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850091919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850105949Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850119388Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850131417Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850144577Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[0]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 343, 343, 319, 333, 337, 330, 343]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850158296Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[0]R generation[0..12]: [184, 188, 188, 189, 189, 191, 191, 177, 188, 189, 187, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850172095Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850185975Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[1]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 343, 343, 319, 333, 337, 330, 343]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850199634Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[1]R generation[0..12]: [184, 188, 188, 189, 189, 191, 191, 177, 188, 189, 187, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850213464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850227223Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[2]R flush_numbers[0..12]: [328, 336, 336, 337, 337, 343, 343, 319, 333, 337, 330, 343]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850241052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[2]R generation[0..12]: [184, 188, 188, 189, 189, 191, 191, 177, 188, 189, 187, 191]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850254681Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850268531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:25.85028342Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850338157Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850378845Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850409074Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"":"downstairs","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850438372Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-26T22:11:25.850465471Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"0","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.85049334Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"1","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850513579Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"client":"2","":"downstairs","session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850528398Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850540897Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"d4ae14ae-f80e-4156-9cde-e1a7eb0cff3e is now active with session: fa47f522-a9b4-418f-8f38-669129cf0b5b","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850553697Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850565926Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950,"session_id":"fa47f522-a9b4-418f-8f38-669129cf0b5b"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:25.850578355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 40960 data: 15 11 11 11 12 17 15 16 16 16 02/30 Read block 89 len 4096 03/30 Read block 343 len 40960 04/30 Read block 152 len 28672 05/30 Read block 547 len 20480 06/30 Read block 478 len 24576 07/30 Write block 286 len 24576 data: 14 16 20 18 18 14 08/30 Flush 09/30 Write block 97 len 24576 data: 15 15 13 12 14 14 10/30 Read block 300 len 24576 11/30 Write block 187 len 16384 data: 14 14 12 11 12/30 Read block 70 len 24576 13/30 Flush 14/30 Flush 15/30 Write block 221 len 36864 data: 15 16 16 18 19 22 22 18 19 16/30 Read block 521 len 36864 17/30 Read block 564 len 40960 18/30 Write block 463 len 28672 data: 19 17 18 19 18 11 15 19/30 Read block 77 len 32768 20/30 Flush 21/30 Write block 350 len 4096 data: 17 22/30 Write block 289 len 12288 data: 19 19 15 23/30 Read block 512 len 8192 24/30 Write block 268 len 16384 data: 14 15 14 14 25/30 Read block 278 len 32768 26/30 Write block 392 len 32768 data: 14 13 11 10 11 11 12 11 27/30 Write block 592 len 32768 data: 11 13 12 13 10 9 6 3 28/30 Read block 409 len 28672 29/30 Read block 266 len 36864 30/30 Write block 323 len 16384 data: 15 17 17 15 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:25.871311806Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 192 192 192 345 345 345 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 192 192 192 347 347 347 F F F 12 240-259 191 191 191 343 343 343 F F F 13 260-279 181 181 181 324 324 324 T T T 14 280-299 192 192 192 344 344 344 T T T 15 300-319 190 190 190 342 342 342 F F F 16 320-339 190 190 190 338 338 338 T T T 17 340-359 191 191 191 343 343 343 T T T 18 360-379 191 191 191 343 343 343 F F F 19 380-399 188 188 188 334 334 334 T T T 20 400-419 192 192 192 344 344 344 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 192 192 192 347 347 347 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 F F F 26 520-539 190 190 190 342 342 342 F F F 27 540-559 190 190 190 342 342 342 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 190 190 190 341 341 341 T T T Max gen: 192, Max flush: 347 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 192 192 192 345 345 345 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 177 177 177 319 319 319 F F F 8 160-179 188 188 188 333 333 333 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 192 192 192 347 347 347 F F F 12 240-259 191 191 191 343 343 343 F F F 13 260-279 193 193 193 348 348 348 F F F 14 280-299 193 193 193 348 348 348 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 193 193 193 348 348 348 F F F 17 340-359 193 193 193 348 348 348 F F F 18 360-379 191 191 191 343 343 343 F F F 19 380-399 193 193 193 348 348 348 F F F 20 400-419 192 192 192 344 344 344 F F F 21 420-439 187 187 187 330 330 330 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 192 192 192 347 347 347 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 F F F 26 520-539 190 190 190 342 342 342 F F F 27 540-559 190 190 190 342 342 342 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 193 193 193 348 348 348 F F F Max gen: 193, Max flush: 348 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.1490011Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.193812936Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.194479923Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.194518082Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.194532351Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.19454666Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} {"msg":"Crucible f3f3a5d8-405f-4dd3-a653-b224e959c440 has session id: 9006c163-18c1-45d8-90ff-229cbf395398","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.195019707Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Upstairs opts: Upstairs UUID: f3f3a5d8-405f-4dd3-a653-b224e959c440, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.195065895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Crucible stats registered with UUID: f3f3a5d8-405f-4dd3-a653-b224e959c440","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.195094693Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225383534Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225411053Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} {"msg":"f3f3a5d8-405f-4dd3-a653-b224e959c440 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225475039Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225494458Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225516777Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225530927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225544256Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225567685Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225595134Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225621702Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225642151Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22566834Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_connection connected from Ok(127.0.0.1:33625)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225770295Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_connection connected from Ok(127.0.0.1:61603)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225808453Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_connection connected from Ok(127.0.0.1:43523)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.225835852Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226510249Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226538407Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226694739Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226727108Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226755616Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.226781625Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22729581Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.227332598Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.227404355Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.227873672Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22790211Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22791667Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.227992886Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228026004Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228040264Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228056912Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228070932Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[0]R flush_numbers[0..12]: [328, 336, 336, 337, 345, 345, 343, 319, 333, 345, 330, 347]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228084921Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[0]R generation[0..12]: [184, 188, 188, 189, 192, 192, 191, 177, 188, 192, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228099041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22811302Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[1]R flush_numbers[0..12]: [328, 336, 336, 337, 345, 345, 343, 319, 333, 345, 330, 347]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228126779Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[1]R generation[0..12]: [184, 188, 188, 189, 192, 192, 191, 177, 188, 192, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228140448Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228154368Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[2]R flush_numbers[0..12]: [328, 336, 336, 337, 345, 345, 343, 319, 333, 345, 330, 347]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228167957Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[2]R generation[0..12]: [184, 188, 188, 189, 192, 192, 191, 177, 188, 192, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228181677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228195536Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:30.228208465Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228228714Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228241803Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228256753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"":"downstairs","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228270112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228282292Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"0","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228295311Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"1","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22830847Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"client":"2","":"downstairs","session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.22832165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228333909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"f3f3a5d8-405f-4dd3-a653-b224e959c440 is now active with session: 9006c163-18c1-45d8-90ff-229cbf395398","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228346838Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228358918Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958,"session_id":"9006c163-18c1-45d8-90ff-229cbf395398"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:30.228371167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 24576 data: 18 17 16 15 14 16 02/30 Write block 35 len 20480 data: 22 23 22 18 16 03/30 Read block 238 len 8192 04/30 Read block 307 len 32768 05/30 Read block 46 len 36864 06/30 Flush 07/30 Write block 423 len 24576 data: 10 9 10 10 8 10 08/30 Write block 549 len 4096 data: 13 09/30 Write block 403 len 40960 data: 14 14 11 10 13 13 16 12 12 12 10/30 Write block 385 len 40960 data: 13 11 11 9 10 13 13 15 14 12 11/30 Read block 109 len 12288 12/30 Read block 66 len 12288 13/30 Flush 14/30 Write block 465 len 40960 data: 19 20 19 12 16 15 13 10 11 12 15/30 Write block 226 len 24576 data: 23 23 19 20 19 18 16/30 Read block 461 len 4096 17/30 Read block 184 len 8192 18/30 Write block 429 len 28672 data: 6 7 9 10 10 9 11 19/30 Flush 20/30 Write block 147 len 20480 data: 15 12 10 13 13 21/30 Flush 22/30 Write block 247 len 24576 data: 15 15 15 18 18 16 23/30 Read block 292 len 24576 24/30 Write block 532 len 12288 data: 12 9 8 25/30 Read block 471 len 28672 26/30 Read block 259 len 8192 27/30 Write block 162 len 4096 data: 17 28/30 Read block 394 len 40960 29/30 Write block 139 len 4096 data: 16 30/30 Read block 174 len 16384 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 9 7 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:30.247369013Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 194 194 194 349 349 349 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 192 192 192 345 345 345 F F F 6 120-139 191 191 191 343 343 343 T T T 7 140-159 194 194 194 352 352 352 F F F 8 160-179 188 188 188 333 333 333 T T T 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 194 194 194 351 351 351 F F F 12 240-259 191 191 191 343 343 343 T T T 13 260-279 193 193 193 348 348 348 F F F 14 280-299 193 193 193 348 348 348 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 193 193 193 348 348 348 F F F 17 340-359 193 193 193 348 348 348 F F F 18 360-379 194 194 194 349 349 349 F F F 19 380-399 194 194 194 350 350 350 F F F 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 F F F 26 520-539 190 190 190 342 342 342 T T T 27 540-559 194 194 194 350 350 350 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 193 193 193 348 348 348 F F F Max gen: 194, Max flush: 352 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 F F F 1 020-039 194 194 194 349 349 349 F F F 2 040-059 188 188 188 336 336 336 F F F 3 060-079 189 189 189 337 337 337 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 192 192 192 345 345 345 F F F 6 120-139 195 195 195 353 353 353 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 195 195 195 353 353 353 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 194 194 194 351 351 351 F F F 12 240-259 195 195 195 353 353 353 F F F 13 260-279 193 193 193 348 348 348 F F F 14 280-299 193 193 193 348 348 348 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 193 193 193 348 348 348 F F F 17 340-359 193 193 193 348 348 348 F F F 18 360-379 194 194 194 349 349 349 F F F 19 380-399 194 194 194 350 350 350 F F F 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 187 187 187 330 330 330 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 F F F 26 520-539 195 195 195 353 353 353 F F F 27 540-559 194 194 194 350 350 350 F F F 28 560-579 190 190 190 341 341 341 F F F 29 580-599 193 193 193 348 348 348 F F F Max gen: 195, Max flush: 353 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.516098879Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.561735765Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562363165Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562395313Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562409052Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562423112Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} {"msg":"Crucible 25cd0250-8caa-460b-8104-3cc6d0c89af6 has session id: 21a35b36-26e2-42fb-bf9c-c48fca51ef86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562849391Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Upstairs opts: Upstairs UUID: 25cd0250-8caa-460b-8104-3cc6d0c89af6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562895159Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Crucible stats registered with UUID: 25cd0250-8caa-460b-8104-3cc6d0c89af6","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.562937636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.59377543Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593812118Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} {"msg":"25cd0250-8caa-460b-8104-3cc6d0c89af6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593849636Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593867696Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593896474Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593914423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593931322Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593947812Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593964171Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.59398051Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.593997639Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.594026558Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_connection connected from Ok(127.0.0.1:54437)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.594174301Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_connection connected from Ok(127.0.0.1:64331)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.594223948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_connection connected from Ok(127.0.0.1:51746)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.594324183Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595024219Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595063757Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595094675Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595132193Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595157562Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595172271Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595633589Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595740753Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.595764962Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596157463Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596198661Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.59622846Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596293066Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596332954Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596357423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596370893Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596384732Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[0]R flush_numbers[0..12]: [328, 349, 336, 337, 345, 345, 353, 352, 353, 345, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596399531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[0]R generation[0..12]: [184, 194, 188, 189, 192, 192, 195, 194, 195, 192, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.59641741Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596432489Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[1]R flush_numbers[0..12]: [328, 349, 336, 337, 345, 345, 353, 352, 353, 345, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596446539Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[1]R generation[0..12]: [184, 194, 188, 189, 192, 192, 195, 194, 195, 192, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596460348Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596474367Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[2]R flush_numbers[0..12]: [328, 349, 336, 337, 345, 345, 353, 352, 353, 345, 330, 351]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596505046Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[2]R generation[0..12]: [184, 194, 188, 189, 192, 192, 195, 194, 195, 192, 187, 194]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2024-11-26T22:11:34.596519705Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596549394Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596575192Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596592942Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596606041Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.59661911Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"":"downstairs","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.5966325Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596644829Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"0","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596660948Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"1","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596674278Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"client":"2","":"downstairs","session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596687727Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596703816Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"25cd0250-8caa-460b-8104-3cc6d0c89af6 is now active with session: 21a35b36-26e2-42fb-bf9c-c48fca51ef86","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596716556Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596728485Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966,"session_id":"21a35b36-26e2-42fb-bf9c-c48fca51ef86"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:34.596740744Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 294 len 12288 data: 14 17 16 02/30 Write block 361 len 20480 data: 15 15 14 15 19 03/30 Read block 138 len 32768 04/30 Read block 280 len 40960 05/30 Read block 491 len 32768 06/30 Read block 436 len 12288 07/30 Read block 418 len 4096 08/30 Write block 444 len 40960 data: 25 24 24 21 18 12 8 8 5 6 09/30 Write block 259 len 16384 data: 14 19 16 18 10/30 Read block 455 len 36864 11/30 Flush 12/30 Write block 239 len 24576 data: 16 14 16 19 20 18 13/30 Read block 156 len 36864 14/30 Write block 119 len 20480 data: 13 12 11 12 11 15/30 Write block 54 len 36864 data: 17 14 18 18 19 18 17 16 18 16/30 Read block 7 len 8192 17/30 Write block 14 len 12288 data: 16 13 14 18/30 Write block 570 len 24576 data: 19 18 18 20 19 15 19/30 Write block 388 len 16384 data: 10 11 14 14 20/30 Read block 79 len 36864 21/30 Read block 138 len 24576 22/30 Read block 573 len 36864 23/30 Write block 338 len 20480 data: 14 14 14 14 13 24/30 Write block 29 len 32768 data: 7 9 8 11 12 16 23 24 25/30 Read block 473 len 8192 26/30 Read block 344 len 8192 27/30 Read block 127 len 24576 28/30 Read block 308 len 20480 29/30 Write block 502 len 32768 data: 14 14 13 16 13 13 13 15 30/30 Read block 137 len 4096 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 19 17 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:34.616775979Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 328 328 328 T T T 1 020-039 194 194 194 349 349 349 T T T 2 040-059 188 188 188 336 336 336 T T T 3 060-079 189 189 189 337 337 337 T T T 4 080-099 192 192 192 345 345 345 F F F 5 100-119 192 192 192 345 345 345 T T T 6 120-139 195 195 195 353 353 353 T T T 7 140-159 194 194 194 352 352 352 F F F 8 160-179 195 195 195 353 353 353 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 194 194 194 351 351 351 T T T 12 240-259 196 196 196 354 354 354 T T T 13 260-279 196 196 196 354 354 354 F F F 14 280-299 196 196 196 354 354 354 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 193 193 193 348 348 348 T T T 17 340-359 193 193 193 348 348 348 T T T 18 360-379 196 196 196 354 354 354 F F F 19 380-399 194 194 194 350 350 350 T T T 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 196 196 196 354 354 354 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 191 191 191 343 343 343 T T T 26 520-539 195 195 195 353 353 353 F F F 27 540-559 194 194 194 350 350 350 F F F 28 560-579 190 190 190 341 341 341 T T T 29 580-599 193 193 193 348 348 348 F F F Max gen: 196, Max flush: 354 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 355 355 355 F F F 1 020-039 197 197 197 355 355 355 F F F 2 040-059 197 197 197 355 355 355 F F F 3 060-079 197 197 197 355 355 355 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 197 197 197 355 355 355 F F F 6 120-139 197 197 197 355 355 355 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 195 195 195 353 353 353 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 197 197 197 355 355 355 F F F 12 240-259 197 197 197 355 355 355 F F F 13 260-279 196 196 196 354 354 354 F F F 14 280-299 196 196 196 354 354 354 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 197 197 197 355 355 355 F F F 17 340-359 197 197 197 355 355 355 F F F 18 360-379 196 196 196 354 354 354 F F F 19 380-399 197 197 197 355 355 355 F F F 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 196 196 196 354 354 354 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 197 197 197 355 355 355 F F F 26 520-539 195 195 195 353 353 353 F F F 27 540-559 194 194 194 350 350 350 F F F 28 560-579 197 197 197 355 355 355 F F F 29 580-599 193 193 193 348 348 348 F F F Max gen: 197, Max flush: 355 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.935235889Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.980273564Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.980937771Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6bdc5ae0b2033c80bbfa3a59474c7c52737638e4\",\n git_commit_timestamp: \"2024-11-26T21:48:03.000000000Z\",\n git_branch: \"mkeeter/defer-flush-and-barrier\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.980975989Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.980990329Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.981004518Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} {"msg":"Crucible 837c41cf-1a9d-46a8-9797-8183070369fe has session id: c618517b-5b8d-4e2f-9342-a31637fc8a23","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.981479405Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Upstairs opts: Upstairs UUID: 837c41cf-1a9d-46a8-9797-8183070369fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.981524993Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Crucible stats registered with UUID: 837c41cf-1a9d-46a8-9797-8183070369fe","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:38.981549571Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.012813554Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.012852362Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} {"msg":"837c41cf-1a9d-46a8-9797-8183070369fe active request set","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.01288612Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.01290813Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.012932968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.012963927Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.012998445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013034733Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013070311Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.0131058Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013139168Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013165257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_connection connected from Ok(127.0.0.1:57741)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013200145Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_connection connected from Ok(127.0.0.1:42257)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013233423Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_connection connected from Ok(127.0.0.1:51687)","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013339968Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.013981167Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014011115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014026224Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014040404Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014057403Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014071452Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014741379Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014771798Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.014787427Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015365109Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015406257Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015432445Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015453124Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015476663Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015499782Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015522131Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.01554519Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 355, 355, 345, 355, 355, 352, 353, 345, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015568929Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[0]R generation[0..12]: [197, 197, 197, 197, 192, 197, 197, 194, 195, 192, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015593298Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015624096Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 355, 355, 345, 355, 355, 352, 353, 345, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015651895Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[1]R generation[0..12]: [197, 197, 197, 197, 192, 197, 197, 194, 195, 192, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015678473Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-26T22:11:39.015708172Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 355, 355, 345, 355, 355, 352, 353, 345, 330, 355]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.01574531Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[2]R generation[0..12]: [197, 197, 197, 197, 192, 197, 197, 194, 195, 192, 187, 197]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015775509Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015806547Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015835106Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015864344Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Next flush: 356","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015890703Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015917791Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"":"downstairs","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2024-11-26T22:11:39.01595042Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015972909Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"0","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.015987528Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"1","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016003887Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"client":"2","":"downstairs","session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016017677Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016030226Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"837c41cf-1a9d-46a8-9797-8183070369fe is now active with session: c618517b-5b8d-4e2f-9342-a31637fc8a23","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016042925Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016055115Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974,"session_id":"c618517b-5b8d-4e2f-9342-a31637fc8a23"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-11-26T22:11:39.016067464Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 20480 data: 16 15 16 20 18 02/30 Read block 130 len 40960 03/30 Write block 242 len 4096 data: 20 04/30 Read block 380 len 32768 05/30 Read block 354 len 16384 06/30 Write block 550 len 20480 data: 11 10 9 8 9 07/30 Read block 410 len 12288 08/30 Write block 189 len 16384 data: 13 12 13 12 09/30 Write block 591 len 8192 data: 12 12 10/30 Read block 287 len 32768 11/30 Write block 498 len 28672 data: 14 14 14 12 15 15 14 12/30 Write block 68 len 4096 data: 13 13/30 Write block 356 len 8192 data: 14 15 14/30 Read block 474 len 24576 15/30 Read block 367 len 32768 16/30 Write block 192 len 12288 data: 13 12 13 17/30 Write block 157 len 16384 data: 20 20 22 19 18/30 Write block 366 len 16384 data: 19 17 16 15 19/30 Write block 527 len 20480 data: 18 15 15 14 12 20/30 Read block 485 len 32768 21/30 Write block 167 len 4096 data: 17 22/30 Write block 41 len 4096 data: 19 23/30 Read block 478 len 8192 24/30 Flush 25/30 Write block 448 len 40960 data: 19 13 9 9 6 7 7 11 11 13 26/30 Flush 27/30 Write block 532 len 32768 data: 13 10 9 13 12 12 12 17 28/30 Read block 257 len 16384 29/30 Write block 240 len 16384 data: 15 17 21 21 30/30 Read block 387 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-26T22:11:39.034068919Z","hostname":"w-01JDN7ZR3FD6ZVPHBBZ6TC0CAH","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 355 355 355 F F F 1 020-039 197 197 197 355 355 355 F F F 2 040-059 198 198 198 356 356 356 F F F 3 060-079 198 198 198 356 356 356 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 197 197 197 355 355 355 F F F 6 120-139 197 197 197 355 355 355 F F F 7 140-159 198 198 198 356 356 356 F F F 8 160-179 198 198 198 356 356 356 F F F 9 180-199 198 198 198 356 356 356 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 197 197 197 355 355 355 F F F 12 240-259 198 198 198 356 356 356 T T T 13 260-279 196 196 196 354 354 354 F F F 14 280-299 196 196 196 354 354 354 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 197 197 197 355 355 355 F F F 17 340-359 198 198 198 356 356 356 F F F 18 360-379 198 198 198 356 356 356 F F F 19 380-399 197 197 197 355 355 355 F F F 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 198 198 198 357 357 357 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 198 198 198 356 356 356 F F F 25 500-519 198 198 198 356 356 356 F F F 26 520-539 198 198 198 356 356 356 T T T 27 540-559 198 198 198 356 356 356 F F F 28 560-579 197 197 197 355 355 355 F F F 29 580-599 198 198 198 356 356 356 F F F Max gen: 198, Max flush: 357 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 355 355 355 F F F 1 020-039 197 197 197 355 355 355 F F F 2 040-059 198 198 198 356 356 356 F F F 3 060-079 198 198 198 356 356 356 F F F 4 080-099 192 192 192 345 345 345 F F F 5 100-119 197 197 197 355 355 355 F F F 6 120-139 197 197 197 355 355 355 F F F 7 140-159 198 198 198 356 356 356 F F F 8 160-179 198 198 198 356 356 356 F F F 9 180-199 198 198 198 356 356 356 F F F 10 200-219 187 187 187 330 330 330 F F F 11 220-239 197 197 197 355 355 355 F F F 12 240-259 199 199 199 358 358 358 F F F 13 260-279 196 196 196 354 354 354 F F F 14 280-299 196 196 196 354 354 354 F F F 15 300-319 190 190 190 342 342 342 F F F 16 320-339 197 197 197 355 355 355 F F F 17 340-359 198 198 198 356 356 356 F F F 18 360-379 198 198 198 356 356 356 F F F 19 380-399 197 197 197 355 355 355 F F F 20 400-419 194 194 194 350 350 350 F F F 21 420-439 194 194 194 351 351 351 F F F 22 440-459 198 198 198 357 357 357 F F F 23 460-479 194 194 194 351 351 351 F F F 24 480-499 198 198 198 356 356 356 F F F 25 500-519 198 198 198 356 356 356 F F F 26 520-539 199 199 199 358 358 358 F F F 27 540-559 198 198 198 356 356 356 F F F 28 560-579 197 197 197 355 355 355 F F F 29 580-599 198 198 198 356 356 356 F F F Max gen: 199, Max flush: 358 7:20 Test duration Test completed real 7:19.859058370 user 54.884040045 sys 17.919719736 trap 0.070816342 tflt 0.038363046 dflt 0.085921718 kflt 0.005517183 lock 11:42:25.070595780 slp 35:34.445511276 lat 5.142303917 stop 0.061249173