{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.887087621Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53828"} {"msg":"region 6c550c9c-0528-43c4-99aa-c4d1d71ed156 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.918311069Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.918486101Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"1f1a541d-799c-4f6c-8130-261803a7ed61","remote_addr":"[fd00:1122:3344:101::a]:53828","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":894,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.918503691Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.927371374Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.927400084Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6c550c9c-0528-43c4-99aa-c4d1d71ed156\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:07.975315124Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"6c550c9c-0528-43c4-99aa-c4d1d71ed156","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.039013725Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"6c550c9c-0528-43c4-99aa-c4d1d71ed156","component":"worker"} {"msg":"region 6c550c9c-0528-43c4-99aa-c4d1d71ed156 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.039074126Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.039157287Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.040567215Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.044623047Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.052893212Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.056085372Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.056303105Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.056358666Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.056401416Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.056440597Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.060691091Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.064695372Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:30:08.204932707Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"4a20a4c6-18a1-45ec-82fe-9a8cd3acd8aa","remote_addr":"[fd00:1122:3344:101::a]:53828","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.564499413Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39656"} {"msg":"region 9942af38-e13f-45a7-8db4-921275f3f406 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.624089881Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.624823831Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.624846761Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"8f1f8ab6-0ee8-4128-809f-1110f85c69eb","remote_addr":"[fd00:1122:3344:101::a]:39656","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":792,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.633796555Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.633817045Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"9942af38-e13f-45a7-8db4-921275f3f406\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/9942af38-e13f-45a7-8db4-921275f3f406\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.689345932Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"9942af38-e13f-45a7-8db4-921275f3f406","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.785632418Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"9942af38-e13f-45a7-8db4-921275f3f406","component":"worker"} {"msg":"region 9942af38-e13f-45a7-8db4-921275f3f406 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.785677418Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.78578228Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.795027108Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.802357871Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.81726156Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.823201786Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9942af38-e13f-45a7-8db4-921275f3f406","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.823417939Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.823470539Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.82351155Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.823549671Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.829490346Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.836660507Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:44.852363997Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"3b71e0ce-49b5-4678-a03d-e736efde5074","remote_addr":"[fd00:1122:3344:101::a]:39656","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.132068407Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42380"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.132132258Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","method":"GET","req_id":"90959269-3750-4c0e-92d5-5552a6006491","remote_addr":"[fd00:1122:3344:101::a]:42380","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":54,"response_code":"200"} {"msg":"path is \"/data/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.132814706Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"dataset is oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.141935413Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"snapshot is oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156@c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.149687121Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156\", \"c45b46bb-a58d-472b-a51d-f5096f5c2f8c\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.149722732Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.160985595Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156/snapshots/c45b46bb-a58d-472b-a51d-f5096f5c2f8c","method":"GET","req_id":"ec7d36b2-0f9d-4400-bed4-2722716ccec1","remote_addr":"[fd00:1122:3344:101::a]:42380","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28689,"response_code":"200"} {"msg":"path is \"/data/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.161147797Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"dataset is oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.169761047Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"snapshot is oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156@c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.178549008Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156\", \"c45b46bb-a58d-472b-a51d-f5096f5c2f8c\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.178581759Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 6c550c9c-0528-43c4-99aa-c4d1d71ed156-c45b46bb-a58d-472b-a51d-f5096f5c2f8c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.187980409Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"applying SMF actions for region 6c550c9c-0528-43c4-99aa-c4d1d71ed156 running snapshot c45b46bb-a58d-472b-a51d-f5096f5c2f8c (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.188147971Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.188200891Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156/snapshots/c45b46bb-a58d-472b-a51d-f5096f5c2f8c/run","method":"POST","req_id":"64e33c70-ddd0-46a1-92f6-55dc8b600d49","remote_addr":"[fd00:1122:3344:101::a]:42380","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27083,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-6c550c9c-0528-43c4-99aa-c4d1d71ed156-c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.190987247Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6c550c9c-0528-43c4-99aa-c4d1d71ed156-c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.199288763Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6c550c9c-0528-43c4-99aa-c4d1d71ed156-c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.218510037Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c550c9c-0528-43c4-99aa-c4d1d71ed156/.zfs/snapshot/c45b46bb-a58d-472b-a51d-f5096f5c2f8c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.218817531Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.218870522Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.218917362Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.218959093Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.219012564Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.22581758Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.233153344Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"region 6c550c9c-0528-43c4-99aa-c4d1d71ed156 running snapshot c45b46bb-a58d-472b-a51d-f5096f5c2f8c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:46.233165844Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.545034461Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39521"} {"msg":"region 3c5eb234-4aae-4edd-96d8-7600de4d039b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.545173963Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.545325545Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.545335145Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"a8dc114b-e74c-474a-bd3c-c9a5aaa22422","remote_addr":"[fd00:1122:3344:101::a]:39521","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":202,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.554054436Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.554095756Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3c5eb234-4aae-4edd-96d8-7600de4d039b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.618382685Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"3c5eb234-4aae-4edd-96d8-7600de4d039b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.725254785Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"3c5eb234-4aae-4edd-96d8-7600de4d039b","component":"worker"} {"msg":"region 3c5eb234-4aae-4edd-96d8-7600de4d039b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.725305236Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.725412617Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.726523701Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.730254099Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.744426569Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.750394005Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.750599058Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.750657089Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.750701409Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.7507403Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.756374441Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.775835589Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:35:49.797192Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions","method":"POST","req_id":"6a4de6f2-631e-4514-a04f-d33dd714bea4","remote_addr":"[fd00:1122:3344:101::a]:39521","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.092915725Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51143"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.093094247Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","method":"GET","req_id":"cd6dd7ea-45b0-45e7-9609-587928d14666","remote_addr":"[fd00:1122:3344:101::a]:51143","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":"200"} {"msg":"path is \"/data/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.093193238Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"dataset is oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.104759636Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"region 3c5eb234-4aae-4edd-96d8-7600de4d039b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.113340725Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.113554158Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.113620538Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","method":"DELETE","req_id":"1f0b572e-a729-4f1b-9fb0-a7079068901d","remote_addr":"[fd00:1122:3344:101::a]:51143","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20477,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.113788451Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","method":"GET","req_id":"4be7a995-d7da-46f6-b94f-877583dd449a","remote_addr":"[fd00:1122:3344:101::a]:51143","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-3c5eb234-4aae-4edd-96d8-7600de4d039b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.11528421Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.12552917Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_cd12cf78-324e-45e0-9823-2a4f43bd78e0/crucible/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.135397436Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"region":"3c5eb234-4aae-4edd-96d8-7600de4d039b","component":"worker"} {"msg":"region 3c5eb234-4aae-4edd-96d8-7600de4d039b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.189436524Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-02T19:36:14.190151863Z","hostname":"oxz_crucible_feea558d-97fc-4a30-98a5-1d3026346001","pid":24114,"uri":"/crucible/0/regions/3c5eb234-4aae-4edd-96d8-7600de4d039b","method":"GET","req_id":"099ed5a7-4eff-49f2-bd92-6f21eff81f82","remote_addr":"[fd00:1122:3344:101::a]:51143","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":"200"}