{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.208443594Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58473"} {"msg":"region ed8e2f31-82b7-4159-8a47-9a729b01e4c3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.290424263Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.30400457Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"a73527f9-d8bf-4c41-b43e-ddcfc3156863","remote_addr":"[fd00:1122:3344:101::a]:58473","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":31631,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.304141674Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.312534825Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.312571716Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ed8e2f31-82b7-4159-8a47-9a729b01e4c3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ed8e2f31-82b7-4159-8a47-9a729b01e4c3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.453686842Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"region":"ed8e2f31-82b7-4159-8a47-9a729b01e4c3","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.611103837Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"7b96b50e-0e1b-45a8-84ac-9fbb70083943","remote_addr":"[fd00:1122:3344:101::a]:58473","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.997968534Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"region":"ed8e2f31-82b7-4159-8a47-9a729b01e4c3","component":"worker"} {"msg":"region ed8e2f31-82b7-4159-8a47-9a729b01e4c3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.998010065Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.998111598Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:45.99857985Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.021680499Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.077503219Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.082546592Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"8f19cab5-c91f-4e42-9628-2bad0234e64b","remote_addr":"[fd00:1122:3344:101::a]:58473","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.10109007Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed8e2f31-82b7-4159-8a47-9a729b01e4c3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.102539258Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.102601029Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.102644481Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.102683572Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.124978189Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:46.173376603Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.335838544Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43119"} {"msg":"region b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.399042778Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.399240873Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.399253403Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"7663eb87-12e4-4480-bf2b-5200330b917e","remote_addr":"[fd00:1122:3344:101::a]:43119","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":277,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.408479396Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.408512417Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.587788998Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"region":"b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:51.72456138Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"4b520d40-ba04-43d9-99c6-d90eaeceb17e","remote_addr":"[fd00:1122:3344:101::a]:43119","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":93,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.068474706Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"region":"b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","component":"worker"} {"msg":"region b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.068510707Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.06863911Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.0690179Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.084456957Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.134726841Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.161190598Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.161520536Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.161579808Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.161634479Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.1616846Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.184306276Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.212685483Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:18:52.393753522Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"6db2f1e4-12c6-41bc-8e0c-b97aa011fb2f","remote_addr":"[fd00:1122:3344:101::a]:43119","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":53,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.763751636Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48601"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.763888889Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","method":"GET","req_id":"a2e4e4bd-7985-456d-9d62-cc93aadc66ff","remote_addr":"[fd00:1122:3344:101::a]:48601","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"path is \"/data/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.764055384Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.776801969Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"region b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.786083804Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.786242938Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.786294199Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","method":"DELETE","req_id":"d3688090-812f-4405-95e8-ba85d5a45277","remote_addr":"[fd00:1122:3344:101::a]:48601","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22258,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.786455063Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","method":"GET","req_id":"6c7f8173-ce6d-4d3e-bb21-b970c57456ab","remote_addr":"[fd00:1122:3344:101::a]:48601","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.787308826Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.802726212Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.812366356Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"region":"b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.85886719Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","method":"GET","req_id":"eda9f9ee-8dbf-4ab8-af8d-0da6933be2e0","remote_addr":"[fd00:1122:3344:101::a]:48601","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"region b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.894087138Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-17T06:19:31.964531373Z","hostname":"oxz_crucible_3e2b6569-1ba1-440f-aed0-249f0de65424","pid":23688,"uri":"/crucible/0/regions/b7f6a6fc-1ed7-40db-a8e2-6c42aeb10bc5","method":"GET","req_id":"62337250-68cd-4921-b28e-820f822133e3","remote_addr":"[fd00:1122:3344:101::a]:48601","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":443,"response_code":"200"}