[ Jul  9 03:52:39 Enabled. ]
[ Jul  9 03:52:39 Rereading configuration. ]
[ Jul  9 03:52:41 Rereading configuration. ]
[ Jul  9 03:52:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:52:43.856338519Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:52:43.860892388Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:52:43.860910208Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:52:43.881554038Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:52:43.978987744Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.480993404Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:58151"}
{"msg":"region fdf6e067-e0b4-4a1f-8c78-83331259d52e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.494144163Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.496479504Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions","method":"POST","req_id":"0ceca900-ec90-4b04-8583-96207f35843f","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":6635,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.497087884Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.507561537Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.507626018Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fdf6e067-e0b4-4a1f-8c78-83331259d52e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.633860766Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"region":"fdf6e067-e0b4-4a1f-8c78-83331259d52e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fdf6e067-e0b4-4a1f-8c78-83331259d52e\" \"--data\" \"/data/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.635992413Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"region":"fdf6e067-e0b4-4a1f-8c78-83331259d52e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.643163618Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions","method":"POST","req_id":"f661c789-3f22-4a49-824a-8eb0b8aa22ff","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":165,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.843028517Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"region":"fdf6e067-e0b4-4a1f-8c78-83331259d52e","component":"worker"}
{"msg":"region fdf6e067-e0b4-4a1f-8c78-83331259d52e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.843114859Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.843345252Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.843744079Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.855629296Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.895208865Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.90066289Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.901395683Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.905562676Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.905612996Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.905649707Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.913961392Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:12.921462392Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.024451045Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions","method":"POST","req_id":"a92945dd-c6c3-4190-9ad7-ceb2fc743759","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.92945381Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","method":"GET","req_id":"d21d075e-e099-41cf-ac16-d1742fd0e8b4","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":95,"response_code":"200"}
{"msg":"path is \"/data/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.931608807Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"dataset is oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.941124463Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"region fdf6e067-e0b4-4a1f-8c78-83331259d52e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.95012297Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.950332983Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.950400834Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","method":"DELETE","req_id":"8ae06c55-22de-4cd1-b242-ca5222bc1081","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18840,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-fdf6e067-e0b4-4a1f-8c78-83331259d52e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.951444593Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.954144989Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","method":"GET","req_id":"1fd0511f-cba2-4be8-ac17-76a60f8ec1c3","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":491,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.95991578Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7d76af44-0180-4c38-8d4d-7b5e4ebfe175/crucible/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:13.968934717Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"region":"fdf6e067-e0b4-4a1f-8c78-83331259d52e","component":"worker"}
{"msg":"region fdf6e067-e0b4-4a1f-8c78-83331259d52e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:14.04608204Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:53:14.091539951Z","hostname":"oxz_crucible_dc0596be-8888-4b39-9302-12ed7ef8b900","pid":21217,"uri":"/crucible/0/regions/fdf6e067-e0b4-4a1f-8c78-83331259d52e","method":"GET","req_id":"5231e7c8-8e01-47f6-9d49-2351f9abfe20","remote_addr":"[fd00:1122:3344:101::c]:58151","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":739,"response_code":"200"}