{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.494670139Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39608"} {"msg":"region 6aa23713-caae-483d-938e-d963293f5fbe state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.49477892Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.495442306Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions","method":"POST","req_id":"d94232bb-30d9-4467-b2b4-8b80d42fc659","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":705,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.495455706Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.504610697Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.504627517Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6aa23713-caae-483d-938e-d963293f5fbe\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6aa23713-caae-483d-938e-d963293f5fbe\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.560540954Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"region":"6aa23713-caae-483d-938e-d963293f5fbe","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6aa23713-caae-483d-938e-d963293f5fbe\" \"--data\" \"/data/regions/6aa23713-caae-483d-938e-d963293f5fbe\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.560574234Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"region":"6aa23713-caae-483d-938e-d963293f5fbe","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.63671957Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"region":"6aa23713-caae-483d-938e-d963293f5fbe","component":"worker"} {"msg":"region 6aa23713-caae-483d-938e-d963293f5fbe state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.636748291Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.636882172Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.659868526Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.670955854Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.681256076Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.684074841Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6aa23713-caae-483d-938e-d963293f5fbe","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.684363604Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.684405574Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.684443374Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.684477834Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.68845019Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.693238002Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:12.805325098Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions","method":"POST","req_id":"d11f26dd-2bdc-4406-9b92-5e4e59619b22","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":83,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.183842149Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e","method":"GET","req_id":"2a9507f6-5aa5-40e0-9a1e-2602dea2a213","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"path is \"/data/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.18737041Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"dataset is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.193750727Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"snapshot is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e@42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.200381951Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\", \"42076beb-0da1-4d38-9020-321413adbb2c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.200403682Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.209640428Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e/snapshots/42076beb-0da1-4d38-9020-321413adbb2c","method":"GET","req_id":"81f82994-9a63-42ee-867b-b95ccce924df","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22288,"response_code":200} {"msg":"path is \"/data/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.21548264Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"dataset is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.222566773Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"snapshot is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e@42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.228742887Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\", \"42076beb-0da1-4d38-9020-321413adbb2c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.228765278Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 64b00ab7-4c00-44df-bad1-42d068b23c5e-42076beb-0da1-4d38-9020-321413adbb2c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.236400475Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.236542857Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e/snapshots/42076beb-0da1-4d38-9020-321413adbb2c/run","method":"POST","req_id":"e8282e0a-f802-4c26-a0b7-9a0b6b631d7b","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21080,"response_code":200} {"msg":"applying SMF actions for region 64b00ab7-4c00-44df-bad1-42d068b23c5e running snapshot 42076beb-0da1-4d38-9020-321413adbb2c (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.236557627Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-64b00ab7-4c00-44df-bad1-42d068b23c5e-42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.241126208Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-64b00ab7-4c00-44df-bad1-42d068b23c5e-42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.248593264Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-64b00ab7-4c00-44df-bad1-42d068b23c5e-42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.257902386Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e/.zfs/snapshot/42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.258247449Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.25829217Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.25833339Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.258368681Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.258412471Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.261380327Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.265888147Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"worker"} {"msg":"region 64b00ab7-4c00-44df-bad1-42d068b23c5e running snapshot 42076beb-0da1-4d38-9020-321413adbb2c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.265897928Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"path is \"/data/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.466490578Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"dataset is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.472570012Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"datafile"} {"msg":"snapshot is oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e@42076beb-0da1-4d38-9020-321413adbb2c","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.479487413Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_84e98c92-e6e0-462c-8ad8-a13b42aceeea/crucible/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e\", \"42076beb-0da1-4d38-9020-321413adbb2c\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.479503833Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T19:39:13.4848202Z","hostname":"oxz_crucible_b656a3ee-e3eb-4b96-9b14-6ed8cd370cd9","pid":23474,"uri":"/crucible/0/regions/64b00ab7-4c00-44df-bad1-42d068b23c5e/snapshots/42076beb-0da1-4d38-9020-321413adbb2c/run","method":"POST","req_id":"d676bfcb-afb0-4ab7-b59e-899f45861e77","remote_addr":"[fd00:1122:3344:101::a]:39608","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18364,"response_code":200}