{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.275825353Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:36150"} {"msg":"region bbd5b1af-1107-4333-8690-420777c54369 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.320793117Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.324051344Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.324069804Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions","method":"POST","req_id":"4ed87fcf-056c-410c-8b29-c69f5df6275d","remote_addr":"[fd00:1122:3344:101::c]:36150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19323,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.335702237Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.335719277Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bbd5b1af-1107-4333-8690-420777c54369\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bbd5b1af-1107-4333-8690-420777c54369\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.387912207Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"bbd5b1af-1107-4333-8690-420777c54369","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bbd5b1af-1107-4333-8690-420777c54369\" \"--data\" \"/data/regions/bbd5b1af-1107-4333-8690-420777c54369\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.390246448Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"bbd5b1af-1107-4333-8690-420777c54369","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.599993963Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions","method":"POST","req_id":"2e458648-6efb-49a9-97c9-f8e17829b4d2","remote_addr":"[fd00:1122:3344:101::c]:36150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":67,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.749753733Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"bbd5b1af-1107-4333-8690-420777c54369","component":"worker"} {"msg":"region bbd5b1af-1107-4333-8690-420777c54369 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.749794904Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.749894276Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.750266412Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.762505476Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.786718938Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.789813502Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bbd5b1af-1107-4333-8690-420777c54369","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.790510394Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.790577795Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.790620456Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.790661066Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.795907398Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:02.800741362Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:03.295980493Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions","method":"POST","req_id":"e55bb624-b6ae-44e9-8f7d-7a7bd94e5f23","remote_addr":"[fd00:1122:3344:101::c]:36150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region 72d28dc7-c7ce-44fa-8317-1e68dfca1909 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.644667488Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.6448328Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions","method":"POST","req_id":"1ae91f9a-e400-47b7-9512-4137c739d4b6","remote_addr":"[fd00:1122:3344:101::c]:36150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":219,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.644849641Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.651845673Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.651863253Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"72d28dc7-c7ce-44fa-8317-1e68dfca1909\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.702385704Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"72d28dc7-c7ce-44fa-8317-1e68dfca1909","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"72d28dc7-c7ce-44fa-8317-1e68dfca1909\" \"--data\" \"/data/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.702433604Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"72d28dc7-c7ce-44fa-8317-1e68dfca1909","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.786687603Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"72d28dc7-c7ce-44fa-8317-1e68dfca1909","component":"worker"} {"msg":"region 72d28dc7-c7ce-44fa-8317-1e68dfca1909 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.786725573Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.786830005Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.787244553Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.791025739Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.800143318Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.803479165Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.803680349Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.80372341Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.803759141Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.803793341Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.807550836Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.818447477Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:09.969780364Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions","method":"POST","req_id":"4533f227-a630-4c1e-a530-5985d30a8110","remote_addr":"[fd00:1122:3344:101::c]:36150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.776600607Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:62523"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.776736039Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","method":"GET","req_id":"a4edc9f4-d8de-4c0a-accc-cdfc4b421466","remote_addr":"[fd00:1122:3344:101::c]:62523","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":200} {"msg":"path is \"/data/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.780535286Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"dataset is oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.791812622Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"region 72d28dc7-c7ce-44fa-8317-1e68dfca1909 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.79912945Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.799304623Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","method":"DELETE","req_id":"4fe6d60f-fcbf-4b4b-9cb7-ff7f6a860446","remote_addr":"[fd00:1122:3344:101::c]:62523","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18812,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.799342153Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-72d28dc7-c7ce-44fa-8317-1e68dfca1909 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.799811992Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.803022498Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","method":"GET","req_id":"bfb94a15-bfcb-4bf4-92dd-4bc22a4585c4","remote_addr":"[fd00:1122:3344:101::c]:62523","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":30,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.821576341Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_583d7ff7-2254-4289-814c-b6caa17b2c5f/crucible/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.827890851Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"region":"72d28dc7-c7ce-44fa-8317-1e68dfca1909","component":"worker"} {"msg":"region 72d28dc7-c7ce-44fa-8317-1e68dfca1909 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.924337242Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T13:48:41.95231894Z","hostname":"oxz_crucible_71730881-1bb2-4aea-8c87-393f500eb76a","pid":23445,"uri":"/crucible/0/regions/72d28dc7-c7ce-44fa-8317-1e68dfca1909","method":"GET","req_id":"4ab41020-e1d8-463c-9a07-bbd55bc4a8b1","remote_addr":"[fd00:1122:3344:101::c]:62523","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":53,"response_code":200}