[ Feb 28 07:44:01 Enabled. ]
[ Feb 28 07:44:01 Rereading configuration. ]
[ Feb 28 07:44:04 Rereading configuration. ]
[ Feb 28 07:44:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.949628516Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.950695594Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:07.950709924Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:08.271636936Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.561319987Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:43681"}
{"msg":"region 98068208-67b0-47a6-889c-f7d86294aff4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.567724438Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.570808942Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions","method":"POST","req_id":"cbfa1bd5-101d-49e2-95c1-5166077979cb","remote_addr":"[fd00:1122:3344:101::b]:43681","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3881,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.570873433Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.580164774Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.580219475Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"98068208-67b0-47a6-889c-f7d86294aff4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/98068208-67b0-47a6-889c-f7d86294aff4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.659629141Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"region":"98068208-67b0-47a6-889c-f7d86294aff4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.782222996Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"region":"98068208-67b0-47a6-889c-f7d86294aff4","component":"worker"}
{"msg":"region 98068208-67b0-47a6-889c-f7d86294aff4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.782269997Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.782537051Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.782922968Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.786939788Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.792728238Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions","method":"POST","req_id":"811a48dc-8d3c-4a3f-9828-5955aa80b581","remote_addr":"[fd00:1122:3344:101::b]:43681","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":673,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.796098406Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.802026149Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.802824443Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.802903624Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.802947895Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.802982726Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.80961173Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:47.814652908Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.328772458Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:59862"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.32893161Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions/98068208-67b0-47a6-889c-f7d86294aff4","method":"GET","req_id":"d1f1c140-e754-4561-bea4-0010de42aab2","remote_addr":"[fd00:1122:3344:101::b]:59862","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":109,"response_code":"200"}
{"msg":"path is \"/data/regions/98068208-67b0-47a6-889c-f7d86294aff4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.329116174Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/98068208-67b0-47a6-889c-f7d86294aff4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.338306773Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"region 98068208-67b0-47a6-889c-f7d86294aff4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.347189577Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.34738983Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.347461131Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions/98068208-67b0-47a6-889c-f7d86294aff4","method":"DELETE","req_id":"88adf743-a3b3-4809-a569-11a288f7b115","remote_addr":"[fd00:1122:3344:101::b]:59862","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18367,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-98068208-67b0-47a6-889c-f7d86294aff4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.348460889Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.348847775Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions/98068208-67b0-47a6-889c-f7d86294aff4","method":"GET","req_id":"095910d3-5a3b-4abd-ae16-8db435cfa4a6","remote_addr":"[fd00:1122:3344:101::b]:59862","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.354400432Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/98068208-67b0-47a6-889c-f7d86294aff4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.36524504Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"region":"98068208-67b0-47a6-889c-f7d86294aff4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.418145436Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions/98068208-67b0-47a6-889c-f7d86294aff4","method":"GET","req_id":"06111e81-9e2c-4fd7-ba58-ed102408b43f","remote_addr":"[fd00:1122:3344:101::b]:59862","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"region 98068208-67b0-47a6-889c-f7d86294aff4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.438306206Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T07:44:48.54410201Z","hostname":"oxz_crucible_5f574fe3-dbda-4f4a-bd75-654848099318","pid":21997,"uri":"/crucible/0/regions/98068208-67b0-47a6-889c-f7d86294aff4","method":"GET","req_id":"f5523c42-7fdf-4967-bb27-fd62c7ba866a","remote_addr":"[fd00:1122:3344:101::b]:59862","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":46,"response_code":"200"}