{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.545958405Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65334"} {"msg":"region a12863e7-70ea-44fd-9f0e-a0d3a734a12b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.546076917Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.5469258Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.546934731Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions","method":"POST","req_id":"99b5caac-b22a-4016-a5b3-4d2bcd7899a2","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":889,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.557117052Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.557134822Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a12863e7-70ea-44fd-9f0e-a0d3a734a12b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a12863e7-70ea-44fd-9f0e-a0d3a734a12b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.610781584Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"region":"a12863e7-70ea-44fd-9f0e-a0d3a734a12b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a12863e7-70ea-44fd-9f0e-a0d3a734a12b\" \"--data\" \"/data/regions/a12863e7-70ea-44fd-9f0e-a0d3a734a12b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.610811444Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"region":"a12863e7-70ea-44fd-9f0e-a0d3a734a12b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.697788984Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"region":"a12863e7-70ea-44fd-9f0e-a0d3a734a12b","component":"worker"} {"msg":"region a12863e7-70ea-44fd-9f0e-a0d3a734a12b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.697828695Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.697934567Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.698321223Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.708675047Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.732015637Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.73533899Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a12863e7-70ea-44fd-9f0e-a0d3a734a12b","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.735612874Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.735658965Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.735698076Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.735734487Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.739482136Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.754170069Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:36.893619242Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions","method":"POST","req_id":"1ac849e4-3b94-4f0c-93f2-12dc60a2824c","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.79965961Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions/a7994acc-ce00-437c-b0e7-f336243c12b0","method":"GET","req_id":"3b81ca39-03d9-404f-b17a-24502735662c","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":96,"response_code":200} {"msg":"path is \"/data/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.804219162Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"dataset is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.813457089Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"snapshot is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0@253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.822662495Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\", \"253ecf72-7666-4b35-b386-eebb38d913fb\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.822689725Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.835467508Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions/a7994acc-ce00-437c-b0e7-f336243c12b0/snapshots/253ecf72-7666-4b35-b386-eebb38d913fb","method":"GET","req_id":"4423a7ff-2a56-46cf-92e4-149178cc8eb5","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31292,"response_code":200} {"msg":"path is \"/data/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.840148612Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"dataset is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.849045734Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"snapshot is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0@253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.856568153Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\", \"253ecf72-7666-4b35-b386-eebb38d913fb\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.856596423Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot a7994acc-ce00-437c-b0e7-f336243c12b0-253ecf72-7666-4b35-b386-eebb38d913fb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.867138341Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"applying SMF actions for region a7994acc-ce00-437c-b0e7-f336243c12b0 running snapshot 253ecf72-7666-4b35-b386-eebb38d913fb (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.867330064Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.867340524Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions/a7994acc-ce00-437c-b0e7-f336243c12b0/snapshots/253ecf72-7666-4b35-b386-eebb38d913fb/run","method":"POST","req_id":"7b3346f0-911e-444b-b206-3d7a8b8f559f","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27213,"response_code":200} {"msg":"creating missing snapshot instance snapshot-a7994acc-ce00-437c-b0e7-f336243c12b0-253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.873321839Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a7994acc-ce00-437c-b0e7-f336243c12b0-253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.889701699Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a7994acc-ce00-437c-b0e7-f336243c12b0-253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.90241764Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a7994acc-ce00-437c-b0e7-f336243c12b0/.zfs/snapshot/253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.902883228Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.902963959Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.90303501Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.903106771Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.903201003Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.907565552Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.912539771Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"worker"} {"msg":"region a7994acc-ce00-437c-b0e7-f336243c12b0 running snapshot 253ecf72-7666-4b35-b386-eebb38d913fb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.912561471Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"path is \"/data/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:37.998751809Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"dataset is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:38.005247942Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"datafile"} {"msg":"snapshot is oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0@253ecf72-7666-4b35-b386-eebb38d913fb","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:38.011619093Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16d319c0-05ea-456a-bf65-c4898038bd22/crucible/regions/a7994acc-ce00-437c-b0e7-f336243c12b0\", \"253ecf72-7666-4b35-b386-eebb38d913fb\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:38.011640454Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-19T08:06:38.017587618Z","hostname":"oxz_crucible_101dd3ee-2913-440c-807d-5c2c750e8187","pid":23185,"uri":"/crucible/0/regions/a7994acc-ce00-437c-b0e7-f336243c12b0/snapshots/253ecf72-7666-4b35-b386-eebb38d913fb/run","method":"POST","req_id":"2546a6cb-7811-4225-a2dc-08abf144dde5","remote_addr":"[fd00:1122:3344:101::a]:65334","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18883,"response_code":200}