{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.059688555Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60923"} {"msg":"region ddd17ee8-a338-4135-8709-754cdab526f1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.09779005Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.099232861Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.099305142Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"77442d99-c7cb-41be-8367-5c8cdb930da5","remote_addr":"[fd00:1122:3344:101::a]:60923","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":8796,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.108432355Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.108455996Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ddd17ee8-a338-4135-8709-754cdab526f1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ddd17ee8-a338-4135-8709-754cdab526f1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.163165923Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"ddd17ee8-a338-4135-8709-754cdab526f1","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.253443831Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"6594e52e-d683-45f1-bacd-0f1f5409730b","remote_addr":"[fd00:1122:3344:101::a]:60923","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.305765147Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"ddd17ee8-a338-4135-8709-754cdab526f1","component":"worker"} {"msg":"region ddd17ee8-a338-4135-8709-754cdab526f1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.305786668Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.30590233Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.306320699Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.313260896Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.323295218Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.326564347Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.327238781Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.327310333Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.327360014Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.327402785Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.331107003Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.335854483Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:30:15.998330188Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"aef68c9d-ad73-4033-99ef-78a3144c0376","remote_addr":"[fd00:1122:3344:101::a]:60923","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.274880971Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60556"} {"msg":"region b4b38917-4310-4291-989f-bdde70d84919 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.304353434Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.305343345Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.305355276Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"f79be197-d317-4415-9188-a9dce5decd40","remote_addr":"[fd00:1122:3344:101::a]:60556","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":1040,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.315346087Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.315385388Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b4b38917-4310-4291-989f-bdde70d84919\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b4b38917-4310-4291-989f-bdde70d84919\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.377319757Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"b4b38917-4310-4291-989f-bdde70d84919","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.488969337Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"b4b38917-4310-4291-989f-bdde70d84919","component":"worker"} {"msg":"region b4b38917-4310-4291-989f-bdde70d84919 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.489025468Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.489138271Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.489699322Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.499410018Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.522314892Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.529074285Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b4b38917-4310-4291-989f-bdde70d84919","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.529389322Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.529461483Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.529515424Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.529569576Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.531268791Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"593501a9-b76f-4ec0-91ad-e075c3bcfd78","remote_addr":"[fd00:1122:3344:101::a]:60556","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":90,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.537787379Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:51.552446589Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.767850663Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58148"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.767956875Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/ddd17ee8-a338-4135-8709-754cdab526f1","method":"GET","req_id":"252065be-91e5-44c2-b3ce-de107ce7d570","remote_addr":"[fd00:1122:3344:101::a]:58148","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"path is \"/data/regions/ddd17ee8-a338-4135-8709-754cdab526f1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.76819629Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.776926105Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1@a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.785455725Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1\", \"a886df7f-b8f2-4e61-87bd-1070ae592dc9\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.785492656Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.796481158Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/ddd17ee8-a338-4135-8709-754cdab526f1/snapshots/a886df7f-b8f2-4e61-87bd-1070ae592dc9","method":"GET","req_id":"920c2ce6-657d-4882-96ef-91c117d6c3b9","remote_addr":"[fd00:1122:3344:101::a]:58148","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28290,"response_code":"200"} {"msg":"path is \"/data/regions/ddd17ee8-a338-4135-8709-754cdab526f1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.799315168Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.80697893Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1@a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.814109811Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ddd17ee8-a338-4135-8709-754cdab526f1\", \"a886df7f-b8f2-4e61-87bd-1070ae592dc9\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.814140901Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot ddd17ee8-a338-4135-8709-754cdab526f1-a886df7f-b8f2-4e61-87bd-1070ae592dc9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.822865766Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"applying SMF actions for region ddd17ee8-a338-4135-8709-754cdab526f1 running snapshot a886df7f-b8f2-4e61-87bd-1070ae592dc9 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.823031439Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.8230714Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/ddd17ee8-a338-4135-8709-754cdab526f1/snapshots/a886df7f-b8f2-4e61-87bd-1070ae592dc9/run","method":"POST","req_id":"c412266e-51c1-4a19-8fc9-744cc7c1fb2a","remote_addr":"[fd00:1122:3344:101::a]:58148","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23793,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-ddd17ee8-a338-4135-8709-754cdab526f1-a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.82543323Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ddd17ee8-a338-4135-8709-754cdab526f1-a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.834648309Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ddd17ee8-a338-4135-8709-754cdab526f1-a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852014852Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ddd17ee8-a338-4135-8709-754cdab526f1/.zfs/snapshot/a886df7f-b8f2-4e61-87bd-1070ae592dc9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852314868Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852378Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852422321Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852464762Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.852516432Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.859120062Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.866641031Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"region ddd17ee8-a338-4135-8709-754cdab526f1 running snapshot a886df7f-b8f2-4e61-87bd-1070ae592dc9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:52.866653131Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.906852391Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39791"} {"msg":"region f3efd9b0-3125-4f07-94d9-b0e9965a6fd9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.966796328Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.967055203Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.967074333Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"a3083bb6-c366-4c66-b611-f343b057826f","remote_addr":"[fd00:1122:3344:101::a]:39791","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":342,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.976897761Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:55.976938752Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f3efd9b0-3125-4f07-94d9-b0e9965a6fd9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.067005376Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.096333736Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"9ff1d938-b924-43a5-bdc5-7900f05fb1a9","remote_addr":"[fd00:1122:3344:101::a]:39791","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":91,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.175286325Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","component":"worker"} {"msg":"region f3efd9b0-3125-4f07-94d9-b0e9965a6fd9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.175329436Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.175456989Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.177373389Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.180952055Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.197492085Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.203451301Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.20388061Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.203971492Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.204050043Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.204128605Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.210750575Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.21950942Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:35:56.494733279Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions","method":"POST","req_id":"7ca81eec-9a27-49c1-8bfb-436578836291","remote_addr":"[fd00:1122:3344:101::a]:39791","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.359585377Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35888"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.359679179Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","method":"GET","req_id":"0ed32d9e-f481-4c82-998f-4909e25531d6","remote_addr":"[fd00:1122:3344:101::a]:35888","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"path is \"/data/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.359895874Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.370389906Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"region f3efd9b0-3125-4f07-94d9-b0e9965a6fd9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.379088339Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.379268683Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.379322215Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","method":"DELETE","req_id":"77c7686c-6c93-4c15-ab58-6507e4ca5a79","remote_addr":"[fd00:1122:3344:101::a]:35888","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19467,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.379687892Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","method":"GET","req_id":"9fdfc83d-41d0-498b-9216-0f852334bf47","remote_addr":"[fd00:1122:3344:101::a]:35888","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-f3efd9b0-3125-4f07-94d9-b0e9965a6fd9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.383307919Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.391013552Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.401474553Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"region":"f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","component":"worker"} {"msg":"region f3efd9b0-3125-4f07-94d9-b0e9965a6fd9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.448799073Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-15T22:36:27.458983229Z","hostname":"oxz_crucible_358f6515-8881-4c6e-9f62-6887e6150952","pid":21757,"uri":"/crucible/0/regions/f3efd9b0-3125-4f07-94d9-b0e9965a6fd9","method":"GET","req_id":"3881dc20-91b9-4edb-a449-8c089d385a7c","remote_addr":"[fd00:1122:3344:101::a]:35888","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":622,"response_code":"200"}