{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.633494495Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55156"} {"msg":"region f6bdbea2-4257-4256-aeee-2f1b45609736 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.693444588Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.697085102Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.697101443Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions","method":"POST","req_id":"cda0c04c-2fe9-48cf-9b17-53559e7ba7ae","remote_addr":"[fd00:1122:3344:101::a]:55156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3714,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.706096128Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.706119739Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f6bdbea2-4257-4256-aeee-2f1b45609736\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/f6bdbea2-4257-4256-aeee-2f1b45609736\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.765202528Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"region":"f6bdbea2-4257-4256-aeee-2f1b45609736","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:22.824993108Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions","method":"POST","req_id":"315f86b9-48b4-4438-aba9-d9d3e267d97f","remote_addr":"[fd00:1122:3344:101::a]:55156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":98,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.067018773Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"region":"f6bdbea2-4257-4256-aeee-2f1b45609736","component":"worker"} {"msg":"region f6bdbea2-4257-4256-aeee-2f1b45609736 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.067048833Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.067160515Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.068510425Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.074202351Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.093424111Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.098151962Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f6bdbea2-4257-4256-aeee-2f1b45609736","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.098402655Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.098455196Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.098503507Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.098545558Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.103455722Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.108380366Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:23.541125353Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions","method":"POST","req_id":"b673c14c-5e96-4b32-b1ed-a4f40eaf0d10","remote_addr":"[fd00:1122:3344:101::a]:55156","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.92696603Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62389"} {"msg":"region edaf7a5d-58a2-4ab9-aeda-2dd3904e2338 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.957902615Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.958051038Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.958059968Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions","method":"POST","req_id":"444cf916-459c-4fc0-a2ec-ff43f90ff7df","remote_addr":"[fd00:1122:3344:101::a]:62389","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":867,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.96814006Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:26.96815884Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"edaf7a5d-58a2-4ab9-aeda-2dd3904e2338\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.029513004Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"region":"edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.103314096Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"region":"edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","component":"worker"} {"msg":"region edaf7a5d-58a2-4ab9-aeda-2dd3904e2338 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.103340666Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.103453238Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.104403222Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.107982116Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.117158168Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.121183754Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.121437188Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.121502809Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.12155141Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.121594981Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.122801689Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions","method":"POST","req_id":"5f4b463e-acfd-47af-a9ed-fb6267f1d39a","remote_addr":"[fd00:1122:3344:101::a]:62389","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.125907576Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:27.132064978Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.553896058Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52763"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.554079361Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","method":"GET","req_id":"9f144198-9417-4de2-b714-bffd22657755","remote_addr":"[fd00:1122:3344:101::a]:52763","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":95,"response_code":"200"} {"msg":"path is \"/data/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.554263364Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.568722752Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"region edaf7a5d-58a2-4ab9-aeda-2dd3904e2338 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.577750148Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.577957231Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.577981251Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","method":"DELETE","req_id":"dcbaaded-8cc0-4ad4-8c5d-a496554e3c4a","remote_addr":"[fd00:1122:3344:101::a]:52763","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23761,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.578150554Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","method":"GET","req_id":"b15f9f32-b114-4197-b702-c143d6e257f2","remote_addr":"[fd00:1122:3344:101::a]:52763","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-edaf7a5d-58a2-4ab9-aeda-2dd3904e2338 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.579564335Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.586614041Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.596848995Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"region":"edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","component":"worker"} {"msg":"region edaf7a5d-58a2-4ab9-aeda-2dd3904e2338 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.638413691Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-15T01:20:58.641443797Z","hostname":"oxz_crucible_88b23891-1726-4275-973f-1bfd9c7d86da","pid":23172,"uri":"/crucible/0/regions/edaf7a5d-58a2-4ab9-aeda-2dd3904e2338","method":"GET","req_id":"1ae57898-0602-47c7-bd75-9ff0075d7b1a","remote_addr":"[fd00:1122:3344:101::a]:52763","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}