[ Feb 22 18:31:15 Enabled. ]
[ Feb 22 18:31:15 Rereading configuration. ]
[ Feb 22 18:31:18 Rereading configuration. ]
[ Feb 22 18:31:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.556191019Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.560081435Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.560093705Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:27.809691363Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.932095954Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34755"}
{"msg":"region 705152e8-611d-4413-aeb0-b99956b77768 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.971129645Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.973220326Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions","method":"POST","req_id":"91598d6f-807d-44fa-bc27-3699c2e7c940","remote_addr":"[fd00:1122:3344:101::a]:34755","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":9117,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.973801347Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.982872764Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:27.982916475Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"705152e8-611d-4413-aeb0-b99956b77768\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/705152e8-611d-4413-aeb0-b99956b77768\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.025529926Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"region":"705152e8-611d-4413-aeb0-b99956b77768","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.140358036Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"region":"705152e8-611d-4413-aeb0-b99956b77768","component":"worker"}
{"msg":"region 705152e8-611d-4413-aeb0-b99956b77768 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.140410257Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.140499828Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.140894736Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.145950214Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.15953847Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.162327184Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.169685868Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.16982638Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.169872171Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.169911912Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.174148724Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.178787865Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.184116299Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions","method":"POST","req_id":"6f9bf585-96fa-418e-8d68-3ea30268e92b","remote_addr":"[fd00:1122:3344:101::a]:34755","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.66827201Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39303"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.668394363Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions/705152e8-611d-4413-aeb0-b99956b77768","method":"GET","req_id":"dfff0369-4a03-445e-a6be-b47a4ed7a47a","remote_addr":"[fd00:1122:3344:101::a]:39303","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":104,"response_code":"200"}
{"msg":"path is \"/data/regions/705152e8-611d-4413-aeb0-b99956b77768\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.668540116Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/705152e8-611d-4413-aeb0-b99956b77768","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.677862047Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"region 705152e8-611d-4413-aeb0-b99956b77768 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.686048317Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.68623539Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.686328172Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions/705152e8-611d-4413-aeb0-b99956b77768","method":"DELETE","req_id":"01819a39-f634-4f06-ac58-c03d88d9510b","remote_addr":"[fd00:1122:3344:101::a]:39303","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":17802,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.686502226Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions/705152e8-611d-4413-aeb0-b99956b77768","method":"GET","req_id":"29a2a09f-d269-4d2d-a1b8-153a28914d74","remote_addr":"[fd00:1122:3344:101::a]:39303","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-705152e8-611d-4413-aeb0-b99956b77768 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.687263481Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.691164127Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/705152e8-611d-4413-aeb0-b99956b77768\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.700909837Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"region":"705152e8-611d-4413-aeb0-b99956b77768","component":"worker"}
{"msg":"region 705152e8-611d-4413-aeb0-b99956b77768 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.750136947Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:28.758858017Z","hostname":"oxz_crucible_8721d3e7-595c-4190-8708-03101668f10a","pid":26247,"uri":"/crucible/0/regions/705152e8-611d-4413-aeb0-b99956b77768","method":"GET","req_id":"75c0d2cf-9f36-49c1-886b-ee4fc5eea5f3","remote_addr":"[fd00:1122:3344:101::a]:39303","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":99,"response_code":"200"}