[ Nov 26 23:31:55 Enabled. ]
[ Nov 26 23:31:55 Rereading configuration. ]
[ Nov 26 23:31:57 Rereading configuration. ]
[ Nov 26 23:32:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:01.231198884Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:01.235158441Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:01.235169331Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:01.249738617Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:01.319258486Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.278429912Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38313"}
{"msg":"region d4ca4963-85b9-4d44-9364-0180c982fb09 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.308094789Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.312349259Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.316876881Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"523ead92-7a92-43a2-ab96-594e852d6ec8","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":12559,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.324194429Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.32421761Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d4ca4963-85b9-4d44-9364-0180c982fb09\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d4ca4963-85b9-4d44-9364-0180c982fb09\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.369893136Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"region":"d4ca4963-85b9-4d44-9364-0180c982fb09","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d4ca4963-85b9-4d44-9364-0180c982fb09\" \"--data\" \"/data/regions/d4ca4963-85b9-4d44-9364-0180c982fb09\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.371799414Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"region":"d4ca4963-85b9-4d44-9364-0180c982fb09","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.47849659Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"region":"d4ca4963-85b9-4d44-9364-0180c982fb09","component":"worker"}
{"msg":"region d4ca4963-85b9-4d44-9364-0180c982fb09 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.47854293Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.478651491Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.479132146Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.483656948Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.496699399Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.500125182Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.500904079Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.500966979Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.50100879Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.50104687Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.505131718Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.510497408Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:41.68850639Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions","method":"POST","req_id":"6cf22956-e1b6-4298-9202-17a39d98d771","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.185998462Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","method":"GET","req_id":"640f2994-b905-4b7b-820e-718dc0dc234a","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"path is \"/data/regions/d4ca4963-85b9-4d44-9364-0180c982fb09\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.188646237Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"dataset is oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.19966892Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"region d4ca4963-85b9-4d44-9364-0180c982fb09 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.20935219Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.209510292Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.209580572Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","method":"DELETE","req_id":"1d056c3e-f08f-4a2f-8383-a8c1ce4b2eb5","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20958,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d4ca4963-85b9-4d44-9364-0180c982fb09 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.212656031Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.212679101Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","method":"GET","req_id":"b25d39bb-2817-4585-90b6-cf88b62b1aaf","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.217281614Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_21d0d336-acb8-4c0e-a82f-076d7f8c3997/crucible/regions/d4ca4963-85b9-4d44-9364-0180c982fb09\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.228706541Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"region":"d4ca4963-85b9-4d44-9364-0180c982fb09","component":"worker"}
{"msg":"region d4ca4963-85b9-4d44-9364-0180c982fb09 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.287233747Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-26T23:32:42.322717699Z","hostname":"oxz_crucible_764c2b2a-f302-43c3-a549-1ef474143d83","pid":23504,"uri":"/crucible/0/regions/d4ca4963-85b9-4d44-9364-0180c982fb09","method":"GET","req_id":"731bdff4-296e-4698-8d8f-fa07598a14bb","remote_addr":"[fd00:1122:3344:101::b]:38313","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}