{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.908856887Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57037"} {"msg":"region 4516ea2b-d914-4c40-b86c-e70fd0818eae state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.940346541Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.943143136Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"9c76b305-2c61-4b19-b32d-2a4698fdae93","remote_addr":"[fd00:1122:3344:101::a]:57037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2869,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.943161256Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.952017652Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:33.952059173Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"4516ea2b-d914-4c40-b86c-e70fd0818eae\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.00290576Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"4516ea2b-d914-4c40-b86c-e70fd0818eae","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.130898315Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"4516ea2b-d914-4c40-b86c-e70fd0818eae","component":"worker"} {"msg":"region 4516ea2b-d914-4c40-b86c-e70fd0818eae state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.130951426Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.131070188Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.132608999Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.139687389Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.150869215Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"d73d913e-9e00-4a51-9c23-7366ead618f3","remote_addr":"[fd00:1122:3344:101::a]:57037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":79,"response_code":"200"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.15727792Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.161514001Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.161713308Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.161754323Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.16178994Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.161829381Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.168885047Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:00:34.17911027Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.299682302Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35829"} {"msg":"region 8ebcd679-db0f-4a84-b34b-5d387d6d65d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.299824255Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.300482328Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"fdd29df8-9328-4a0a-8e08-2ba75e6c6e90","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":776,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.300498088Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.311544507Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.311570828Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"8ebcd679-db0f-4a84-b34b-5d387d6d65d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.392104364Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"8ebcd679-db0f-4a84-b34b-5d387d6d65d4","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.477654989Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"c84ca4da-1de6-4b56-a7ef-14917e1a6003","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":145,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.527043648Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"8ebcd679-db0f-4a84-b34b-5d387d6d65d4","component":"worker"} {"msg":"region 8ebcd679-db0f-4a84-b34b-5d387d6d65d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.527074799Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.527195711Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.534417204Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.54126752Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.558293908Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.565724375Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8ebcd679-db0f-4a84-b34b-5d387d6d65d4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.56600943Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.566065291Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.566107572Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.566147373Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.572270214Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:58.584278032Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.11654152Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"847cbba9-9fcc-4154-9762-8b7076ec679c","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.648100834Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50240"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.648788068Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","method":"GET","req_id":"2fc1e56f-d148-4f16-bae0-cf94a6bd6efe","remote_addr":"[fd00:1122:3344:101::a]:50240","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":665,"response_code":"200"} {"msg":"path is \"/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.651204596Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.6599812Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae@2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.669462628Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae\", \"2839f329-0315-4eef-84bf-18eb46bddaa5\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.669497598Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.678598609Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/snapshots/2839f329-0315-4eef-84bf-18eb46bddaa5","method":"GET","req_id":"2f86edf0-bb8a-4f11-ba6d-be37e395963a","remote_addr":"[fd00:1122:3344:101::a]:50240","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29649,"response_code":"200"} {"msg":"path is \"/data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.678764422Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.687711359Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae@2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.696543044Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae\", \"2839f329-0315-4eef-84bf-18eb46bddaa5\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.696576705Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.706011112Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"applying SMF actions for region 4516ea2b-d914-4c40-b86c-e70fd0818eae running snapshot 2839f329-0315-4eef-84bf-18eb46bddaa5 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.706192085Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.706241866Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/snapshots/2839f329-0315-4eef-84bf-18eb46bddaa5/run","method":"POST","req_id":"106c03bc-e585-49f7-a156-82409a6f35a7","remote_addr":"[fd00:1122:3344:101::a]:50240","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27505,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.708981331Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.717210354Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4516ea2b-d914-4c40-b86c-e70fd0818eae-2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.734282742Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4516ea2b-d914-4c40-b86c-e70fd0818eae/.zfs/snapshot/2839f329-0315-4eef-84bf-18eb46bddaa5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.734608269Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.73467185Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.734726451Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.734776722Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.734845333Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.739950774Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.748149457Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"region 4516ea2b-d914-4c40-b86c-e70fd0818eae running snapshot 2839f329-0315-4eef-84bf-18eb46bddaa5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:05:59.748162787Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.315981543Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33658"} {"msg":"region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.347270583Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.347433706Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"f4758631-ecd2-41e6-a9b2-bb9dbb6d089c","remote_addr":"[fd00:1122:3344:101::a]:33658","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":214,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.347456787Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.357253131Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.357285541Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.431724317Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.478853521Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"1a982ac8-fad1-4af4-880d-fa7e03cff3f1","remote_addr":"[fd00:1122:3344:101::a]:33658","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.542348579Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","component":"worker"} {"msg":"region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.542375389Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.542510362Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.543647255Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.54998322Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.561991788Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.568543958Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.568842074Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.568905315Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.568954896Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.569004707Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.57670391Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.593496523Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:02.762338949Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions","method":"POST","req_id":"a188ba43-ab50-4a8d-8c34-88593e6f7a50","remote_addr":"[fd00:1122:3344:101::a]:33658","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.597559045Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39501"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.59777662Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","method":"GET","req_id":"10b7a44c-d10b-449a-9182-d80273c8aa7e","remote_addr":"[fd00:1122:3344:101::a]:39501","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":"200"} {"msg":"path is \"/data/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.597843341Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.608149675Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.616763306Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.617314567Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.617330067Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","method":"DELETE","req_id":"abe14eeb-38ed-407c-8ff1-c73c9aa88a83","remote_addr":"[fd00:1122:3344:101::a]:39501","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19403,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.617351097Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","method":"GET","req_id":"54b464c2-1825-4ac1-b3a0-026308d9b974","remote_addr":"[fd00:1122:3344:101::a]:39501","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.618837637Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.628344605Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.638747981Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"region":"22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","component":"worker"} {"msg":"region 22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.68306855Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T09:06:33.687816474Z","hostname":"oxz_crucible_77a606c0-477e-4aa7-9c0a-8f802aa58471","pid":21382,"uri":"/crucible/0/regions/22e9cdca-3e63-4c37-b6fc-bc5fa4e1bee7","method":"GET","req_id":"80a1a9bb-2166-4836-a669-735ab9d90a0f","remote_addr":"[fd00:1122:3344:101::a]:39501","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":110,"response_code":"200"}