[ Mar 24 22:28:50 Disabled. ]
[ Mar 24 22:28:50 Rereading configuration. ]
[ Mar 24 22:28:51 Enabled. ]
[ Mar 24 22:28:51 Rereading configuration. ]
[ Mar 24 22:28:56 Executing start method ("/opt/oxide/pantry/bin/crucible-pantry run -l [fd00:1122:3344:101::11]:17000"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:28:56.897794491Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412}
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:28:56.9085135Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot"}
{"msg":"listen IP: [fd00:1122:3344:101::11]:17000","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:28:56.996845223Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.026925187Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54928"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.045690603Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"b1d4fd1a-c873-4598-addd-013e260f1b6d","remote_addr":"[fd00:1122:3344:101::c]:54928","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":9011,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.232086349Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:41067"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.232243974Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"9e49d7b5-b05c-4b38-8b2b-13f9683e5d5d","remote_addr":"[fd00:1122:3344:101::c]:41067","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":44,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.255922311Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:41518"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:02.256062575Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"2a090736-78e7-4a88-b021-2611ec20735c","remote_addr":"[fd00:1122:3344:101::c]:41518","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":41,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:03.712649206Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63820"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:03.715557765Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"4dd9299f-b1d1-4271-96d9-d7a628edda73","remote_addr":"[fd00:1122:3344:101::a]:63820","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.251046153Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53966"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.251219709Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"fa679332-5789-4756-a4f3-42c25408d856","remote_addr":"[fd00:1122:3344:101::a]:53966","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":44,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.292951411Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38679"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.294021113Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42569"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.294171168Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"4e1aad99-f618-424a-bca1-faa34c5552d5","remote_addr":"[fd00:1122:3344:101::a]:42569","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":32,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.982695807Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61605"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:04.985188704Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"b0bf5f0e-0676-4272-a5b9-1f73dd6f38f7","remote_addr":"[fd00:1122:3344:101::b]:61605","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":69,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:05.200720903Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61349"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:05.200847927Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"df2044ba-a2b3-48d9-aa12-440132b66fd4","remote_addr":"[fd00:1122:3344:101::b]:61349","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:05.53300306Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:41570"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:05.533105393Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"45df3818-8903-4f4c-8768-16765047b65c","remote_addr":"[fd00:1122:3344:101::b]:41570","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":50,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.074187851Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:34859"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.074255443Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"72348f35-3130-46cb-b65a-ad458393138e","remote_addr":"[fd00:1122:3344:101::c]:34859","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":35,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.249074912Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:62279"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.249171075Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"229c93ba-712e-4bc5-a641-e2475ae87235","remote_addr":"[fd00:1122:3344:101::c]:62279","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.273994277Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43937"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:32.27408319Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"59ce41a0-9c04-40e6-8f2a-03811ad38cae","remote_addr":"[fd00:1122:3344:101::c]:43937","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":24,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:33.749271402Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43795"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:33.749380526Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"a4d8750d-1097-4ece-b252-2154d943007f","remote_addr":"[fd00:1122:3344:101::a]:43795","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:34.262841376Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39282"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:34.262932509Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"ee3b5409-6111-4d6c-a390-31d85a64accc","remote_addr":"[fd00:1122:3344:101::a]:39282","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":25,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:34.301150103Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50880"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:34.301211505Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"65118dac-bb88-4b6f-afd0-08e1a209da77","remote_addr":"[fd00:1122:3344:101::a]:50880","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":18,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.012665158Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:36809"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.01273776Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"aba1def7-3b50-4218-aae4-96a213f3c84c","remote_addr":"[fd00:1122:3344:101::b]:36809","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":50,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.211589538Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:59693"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.211701211Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"a091ed37-554e-4fd1-b1b2-bb759fc0c35d","remote_addr":"[fd00:1122:3344:101::b]:59693","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.545133533Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38612"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:35.545219706Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"f35ab65e-bc5a-439e-9d39-70cc427cedd7","remote_addr":"[fd00:1122:3344:101::b]:38612","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":20,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.180568399Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0","method":"GET","req_id":"e197ab02-b42f-4f1d-b477-cdc62c904f66","remote_addr":"[fd00:1122:3344:101::a]:43795","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.292307292Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33563"}
{"msg":"no entry exists for volume e06c62b0-0a83-4da5-b35a-4218c785c68e, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.304458855Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.314484353Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"7103cd3a3d7b0112d2949dd135db06fef0c156bb\",\n    git_commit_timestamp: \"2025-11-10T22:24:53.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.90.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: false,\n    opt_level: 3,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.314521634Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.314568435Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Using region definition RegionDefinition { block_size: 512, extent_size: Block { value: 131072, shift: 9 }, extent_count: 32, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.314611566Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Crucible e06c62b0-0a83-4da5-b35a-4218c785c68e has session id: 08365fb5-e96d-46dc-85dc-0029f91320de","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.333692543Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Upstairs opts: Upstairs UUID: e06c62b0-0a83-4da5-b35a-4218c785c68e, Targets: [[fd00:1122:3344:101::15]:19000, [fd00:1122:3344:101::19]:19000, [fd00:1122:3344:101::1a]:19000], lossy: false, flush_timeout: None, key populated: true,  cert_pem populated: false,  key_pem populated: false,  root_cert_pem populated: false,  Control: None,  read_only: false","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.333828187Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Crucible stats registered with UUID: e06c62b0-0a83-4da5-b35a-4218c785c68e","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.333873558Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"notify_task started","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.334279111Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"job":"notify","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"client is waiting for oneshot","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.334367053Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"client is waiting for oneshot","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.334394624Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"client is waiting for oneshot","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336376205Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"volume e06c62b0-0a83-4da5-b35a-4218c785c68e constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336511919Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"The guest has requested activation","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.33652406Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"e06c62b0-0a83-4da5-b35a-4218c785c68e active request set","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.33653025Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"sending connect oneshot to client","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336570061Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"sending connect oneshot to client","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336629823Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"sending connect oneshot to client","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336642443Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"connecting to [fd00:1122:3344:101::1a]:19000","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336697125Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"connecting to [fd00:1122:3344:101::15]:19000","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336743056Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"connecting to [fd00:1122:3344:101::19]:19000","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336755237Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"ds_connection connected from Ok([fd00:1122:3344:101::11]:56223)","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336816319Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"ds_connection connected from Ok([fd00:1122:3344:101::11]:44831)","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.33686676Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"ds_connection connected from Ok([fd00:1122:3344:101::11]:40994)","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.336879281Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"io task","client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"downstairs client at Some([fd00:1122:3344:101::1a]:19000) has region UUID 58948e93-11a9-4043-a060-ceda389dd7be","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.339980326Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"downstairs client at Some([fd00:1122:3344:101::15]:19000) has region UUID ab5fd91f-d857-49d4-b0ed-17443042c86d","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.340036918Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.340344907Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.340395489Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"downstairs client at Some([fd00:1122:3344:101::19]:19000) has region UUID 891bb926-8e26-48ec-8289-0b6145723c72","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.343907816Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344184355Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344244947Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344321649Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344333999Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344397732Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344458233Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344517975Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344528045Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344585857Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344641679Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344651819Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344705401Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344711021Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"":"downstairs","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"No downstairs reconciliation required","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344745432Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344779393Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"0","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344784843Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"1","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344818374Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"client":"2","":"downstairs","session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"All required reconciliation work is completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344823814Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344857555Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"e06c62b0-0a83-4da5-b35a-4218c785c68e is now active with session: 08365fb5-e96d-46dc-85dc-0029f91320de","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344891917Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"Set Active after no reconciliation","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344897037Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"session_id":"08365fb5-e96d-46dc-85dc-0029f91320de","component":"datafile"}
{"msg":"The guest has finished waiting for activation","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344930618Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Activated sub_volume e06c62b0-0a83-4da5-b35a-4218c785c68e","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344935688Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"Activated sub_volume 98974f6e-c45f-46be-8723-69301668d43d","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344971369Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"volume e06c62b0-0a83-4da5-b35a-4218c785c68e activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.344976289Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"volume e06c62b0-0a83-4da5-b35a-4218c785c68e constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.34501198Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:29:59.3450172Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0/volume/e06c62b0-0a83-4da5-b35a-4218c785c68e","method":"POST","req_id":"76bea330-2dfb-47b8-a1ed-ddc4cfa4f27b","remote_addr":"[fd00:1122:3344:101::a]:33563","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":51985,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:30:00.071043321Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62249"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:30:00.076491689Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0/volume/e06c62b0-0a83-4da5-b35a-4218c785c68e/bulk-write","method":"POST","req_id":"294570ac-df96-4c6f-bece-0e5a0490d95b","remote_addr":"[fd00:1122:3344:101::a]:62249","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":5361,"response_code":204}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:30:00.641200574Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0/volume/e06c62b0-0a83-4da5-b35a-4218c785c68e/bulk-write","method":"POST","req_id":"70307e9b-de76-467f-8c60-0c4f83c93f49","remote_addr":"[fd00:1122:3344:101::a]:62249","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":1831,"response_code":204}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:30:01.155160391Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0/volume/e06c62b0-0a83-4da5-b35a-4218c785c68e/bulk-write","method":"POST","req_id":"1f297454-c913-4299-a8be-9224612ea93e","remote_addr":"[fd00:1122:3344:101::a]:62249","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":1933,"response_code":204}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2026-03-24T22:30:01.311758061Z","hostname":"oxz_crucible_pantry_0ed80d4c-8988-4df0-8464-b85ce15906f6","pid":24412,"uri":"/crucible/pantry/0/volume/e06c62b0-0a83-4da5-b35a-4218c785c68e/bulk-write","method":"POST","req_id":"29927181-746d-4e53-9e8c-01eaee9655a3","remote_addr":"[fd00:1122:3344:101::a]:62249","local_addr":"[fd00:1122:3344:101::11]:17000","component":"dropshot","latency_us":1190,"response_code":204}