[ Mar  3 23:31:15 Enabled. ]
[ Mar  3 23:31:15 Rereading configuration. ]
[ Mar  3 23:31:17 Rereading configuration. ]
[ Mar  3 23:31:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.564921536Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.568069315Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.568077496Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.580912522Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.746328424Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.093939902Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41314"}
{"msg":"region 72671548-614a-4f44-84a0-d229ea99ec91 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.133058808Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.138600746Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"04d2f66b-68b4-43b3-b0c1-5a8ebf51fc74","remote_addr":"[fd00:1122:3344:101::a]:41314","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20414,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.138623747Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.147920092Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.147949433Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"72671548-614a-4f44-84a0-d229ea99ec91\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/72671548-614a-4f44-84a0-d229ea99ec91\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.208401889Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"72671548-614a-4f44-84a0-d229ea99ec91","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"72671548-614a-4f44-84a0-d229ea99ec91\" \"--data\" \"/data/regions/72671548-614a-4f44-84a0-d229ea99ec91\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.210363615Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"72671548-614a-4f44-84a0-d229ea99ec91","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.32231117Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"c0454ed4-9121-42b9-bb69-5377882339e6","remote_addr":"[fd00:1122:3344:101::a]:41314","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.347248562Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"72671548-614a-4f44-84a0-d229ea99ec91","component":"worker"}
{"msg":"region 72671548-614a-4f44-84a0-d229ea99ec91 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.347271893Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.347382476Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.34785539Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.35626932Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.367324345Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.370216098Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.370799274Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.370870136Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.370911357Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.370949599Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.375408936Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.387486981Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:08.683587303Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"8ed473f3-3223-4dbf-8e68-45915c359ed1","remote_addr":"[fd00:1122:3344:101::a]:41314","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.900738428Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53558"}
{"msg":"region 25aa6a2a-7648-4f76-a3bf-dab8162bf4ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.901387696Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.902308272Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.902319793Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"857e1052-2f31-47e9-bcac-e7833a64f2e7","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1525,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.916035524Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.916066875Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"25aa6a2a-7648-4f76-a3bf-dab8162bf4ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/25aa6a2a-7648-4f76-a3bf-dab8162bf4ca\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.970515889Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"25aa6a2a-7648-4f76-a3bf-dab8162bf4ca\" \"--data\" \"/data/regions/25aa6a2a-7648-4f76-a3bf-dab8162bf4ca\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:16.972318121Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.080260022Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","component":"worker"}
{"msg":"region 25aa6a2a-7648-4f76-a3bf-dab8162bf4ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.080296223Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.080399396Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.080832268Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.087772126Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.100822579Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.103529246Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/25aa6a2a-7648-4f76-a3bf-dab8162bf4ca","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.103727412Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.103780543Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.103821354Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.103860275Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.107827579Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.113039797Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:17.184416665Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"8b7ba8fa-80fe-4900-907c-763899b19b1f","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.416347649Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/72671548-614a-4f44-84a0-d229ea99ec91","method":"GET","req_id":"599b4f7c-f762-42c1-bb87-4ba17eb95c80","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2869,"response_code":"200"}
{"msg":"path is \"/data/regions/72671548-614a-4f44-84a0-d229ea99ec91\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.422424483Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"dataset is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.431111611Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"snapshot is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91@989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.439362516Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91\", \"989ec40f-827e-49bd-8aad-d6e3caa9a32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.439384637Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.45002349Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/72671548-614a-4f44-84a0-d229ea99ec91/snapshots/989ec40f-827e-49bd-8aad-d6e3caa9a32b","method":"GET","req_id":"5be1696b-571b-42b9-9fb7-46b6311016b7","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":30057,"response_code":"200"}
{"msg":"path is \"/data/regions/72671548-614a-4f44-84a0-d229ea99ec91\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.453690505Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"dataset is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.461271411Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"snapshot is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91@989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.468665692Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91\", \"989ec40f-827e-49bd-8aad-d6e3caa9a32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.468717084Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 72671548-614a-4f44-84a0-d229ea99ec91-989ec40f-827e-49bd-8aad-d6e3caa9a32b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.477217556Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"applying SMF actions for region 72671548-614a-4f44-84a0-d229ea99ec91 running snapshot 989ec40f-827e-49bd-8aad-d6e3caa9a32b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.477425382Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.477442983Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/72671548-614a-4f44-84a0-d229ea99ec91/snapshots/989ec40f-827e-49bd-8aad-d6e3caa9a32b/run","method":"POST","req_id":"bfcdba80-c8b2-401f-83ca-ebde6cb7a4ee","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23776,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-72671548-614a-4f44-84a0-d229ea99ec91-989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.479569644Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-72671548-614a-4f44-84a0-d229ea99ec91-989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.488946461Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-72671548-614a-4f44-84a0-d229ea99ec91-989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.498493254Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72671548-614a-4f44-84a0-d229ea99ec91/.zfs/snapshot/989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.498821153Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.498888945Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.498935927Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.498976548Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.499023309Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.503055454Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.507769359Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"region 72671548-614a-4f44-84a0-d229ea99ec91 running snapshot 989ec40f-827e-49bd-8aad-d6e3caa9a32b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.507786909Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"path is \"/data/regions/72671548-614a-4f44-84a0-d229ea99ec91\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.663083052Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"dataset is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.671713198Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"snapshot is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91@989ec40f-827e-49bd-8aad-d6e3caa9a32b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.67947148Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/72671548-614a-4f44-84a0-d229ea99ec91\", \"989ec40f-827e-49bd-8aad-d6e3caa9a32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.679502191Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:18.687388466Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/72671548-614a-4f44-84a0-d229ea99ec91/snapshots/989ec40f-827e-49bd-8aad-d6e3caa9a32b/run","method":"POST","req_id":"8d8416b6-0e6c-4bf7-8b91-8601f31cc063","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24342,"response_code":"200"}
{"msg":"region dea980f8-c18c-4e0a-9f2b-631885e7a07a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.854501802Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.854654246Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"d9c58843-ed40-454e-ae4d-239c9282ce9a","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":207,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.854710498Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.864637281Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.864662712Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dea980f8-c18c-4e0a-9f2b-631885e7a07a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.921886905Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"dea980f8-c18c-4e0a-9f2b-631885e7a07a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dea980f8-c18c-4e0a-9f2b-631885e7a07a\" \"--data\" \"/data/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:22.923553013Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"dea980f8-c18c-4e0a-9f2b-631885e7a07a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.002353902Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"dea980f8-c18c-4e0a-9f2b-631885e7a07a","component":"worker"}
{"msg":"region dea980f8-c18c-4e0a-9f2b-631885e7a07a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.002392583Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.002508376Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.004672248Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.00788887Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.016319451Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.020419168Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.020772688Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.020800099Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.02083256Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.020877731Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.025665137Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.032367389Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:23.092934467Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions","method":"POST","req_id":"9bcbec3d-df8a-47b3-87df-4109d1d4d058","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.394236843Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","method":"GET","req_id":"8b86034a-a709-4207-9286-02342f4c606f","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.397673181Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"dataset is oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.411056223Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"region dea980f8-c18c-4e0a-9f2b-631885e7a07a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.420261776Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.42042099Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.420470382Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","method":"DELETE","req_id":"adf5754d-b684-4055-ba17-045299735697","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22815,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-dea980f8-c18c-4e0a-9f2b-631885e7a07a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.421587764Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.4232079Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","method":"GET","req_id":"85a4b72a-4a61-4eee-b49f-051ea35bc02f","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.428314856Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1eb0bcd8-532b-47d1-8d19-9b3024501cec/crucible/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.439962878Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"region":"dea980f8-c18c-4e0a-9f2b-631885e7a07a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.481904665Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","method":"GET","req_id":"b209c686-6389-4721-9acc-d058ac2bbfd8","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":572,"response_code":"200"}
{"msg":"region dea980f8-c18c-4e0a-9f2b-631885e7a07a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.50379439Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:42:47.565648836Z","hostname":"oxz_crucible_a3567a1b-df9a-41f7-9877-f878cd0af8c9","pid":23800,"uri":"/crucible/0/regions/dea980f8-c18c-4e0a-9f2b-631885e7a07a","method":"GET","req_id":"fcad61ec-6373-4ed0-91cc-699b4d6b3af9","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20,"response_code":"200"}