[ Jul 15 19:54:09 Enabled. ]
[ Jul 15 19:54:09 Rereading configuration. ]
[ Jul 15 19:54:10 Rereading configuration. ]
[ Jul 15 19:54:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:12.398839668Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:12.40346553Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:12.40348498Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:12.426178671Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:12.522243931Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.878251449Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57407"}
{"msg":"region cd9d962a-9ca2-42a8-bf9f-ca72b40babf9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.893283628Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.899772827Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"c7fe66f6-a58e-4ee0-a4c2-7bd39077676d","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14331,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.899801028Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.912689924Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:41.912720195Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.081613592Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"7d72991c-444d-45b9-b280-e3c498d7542f","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"creating region Region { id: RegionId(\"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.167110191Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\" \"--data\" \"/data/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.169037199Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.57361082Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","component":"worker"}
{"msg":"region cd9d962a-9ca2-42a8-bf9f-ca72b40babf9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.573676021Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.573773213Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.574320144Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.592387463Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.698143255Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.72401537Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"7aaca708-91a8-47b1-9185-ca3be921142e","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.74867048Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.749615339Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.753534347Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.753636429Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.75368083Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.850705428Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:42.905382955Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.164216089Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","method":"GET","req_id":"8a7bd41c-0a3a-4842-9ff7-152bf3250ffb","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.167287101Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"dataset is oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.176186058Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"region cd9d962a-9ca2-42a8-bf9f-ca72b40babf9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.184743088Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.18488603Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.184957682Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","method":"DELETE","req_id":"ad54f7b5-706d-407c-ba84-3aca9e910650","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18319,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-cd9d962a-9ca2-42a8-bf9f-ca72b40babf9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.186094564Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.187415471Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","method":"GET","req_id":"113190c6-1587-44df-ac10-4c8184febebf","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.224213192Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.234496006Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.243815222Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","method":"GET","req_id":"9af40d70-5494-4a4d-a1de-f7789176e794","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region cd9d962a-9ca2-42a8-bf9f-ca72b40babf9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.365248796Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:46.38308319Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/cd9d962a-9ca2-42a8-bf9f-ca72b40babf9","method":"GET","req_id":"ec726416-543e-4e64-93d0-883aa381f1e9","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"region e1a4adc6-e1d5-46d3-bb41-f127f8260d3f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:50.965467701Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:50.967532222Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"f35aff1a-198d-40ed-991a-c6382e9d5356","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2126,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:50.967565723Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:50.975361608Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:50.975389528Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.087808023Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\" \"--data\" \"/data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.089628479Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.224724424Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"380b7b43-49f5-401e-8e0f-f670d867123a","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.435007644Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","component":"worker"}
{"msg":"region e1a4adc6-e1d5-46d3-bb41-f127f8260d3f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.435035984Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.435143856Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.436768149Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.450025702Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.495280622Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.509495914Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.509933213Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.510023755Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.510112097Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.510192428Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.528450061Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.5480334Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:54:51.747580777Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"bf37430a-f1e6-4944-9928-4be6be87952c","remote_addr":"[fd00:1122:3344:101::a]:57407","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.863053889Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47612"}
{"msg":"region a24e8017-dd23-4ba3-b37f-3a302d906591 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.863192641Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.863732402Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.863741832Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"2adecdb8-9214-407c-b3b1-9c518ee74c9d","remote_addr":"[fd00:1122:3344:101::a]:47612","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":633,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.87219724Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.872236761Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a24e8017-dd23-4ba3-b37f-3a302d906591\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a24e8017-dd23-4ba3-b37f-3a302d906591\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.918725755Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"a24e8017-dd23-4ba3-b37f-3a302d906591","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a24e8017-dd23-4ba3-b37f-3a302d906591\" \"--data\" \"/data/regions/a24e8017-dd23-4ba3-b37f-3a302d906591\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.918765386Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"a24e8017-dd23-4ba3-b37f-3a302d906591","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.995053522Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"region":"a24e8017-dd23-4ba3-b37f-3a302d906591","component":"worker"}
{"msg":"region a24e8017-dd23-4ba3-b37f-3a302d906591 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.995089393Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.995185755Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:58.996389129Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.007128393Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.022804284Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.026317424Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a24e8017-dd23-4ba3-b37f-3a302d906591","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.02663956Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.026695402Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.026742892Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.026782533Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.030516947Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.036146619Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T19:59:59.150246477Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions","method":"POST","req_id":"c1a3a5a4-348c-4fa7-af7a-b355012894a9","remote_addr":"[fd00:1122:3344:101::a]:47612","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.346392302Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63832"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.346487784Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","method":"GET","req_id":"45b270a8-b30a-4272-b142-a3424dd9a228","remote_addr":"[fd00:1122:3344:101::a]:63832","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"path is \"/data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.346725849Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"dataset is oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.35938116Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"snapshot is oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f@960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.372605783Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\", \"960a2b67-7152-477d-a1d3-11128f024a61\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.372669495Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.385366046Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f/snapshots/960a2b67-7152-477d-a1d3-11128f024a61","method":"GET","req_id":"e54dee97-c357-44b3-874c-381e5f80db41","remote_addr":"[fd00:1122:3344:101::a]:63832","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":38680,"response_code":"200"}
{"msg":"path is \"/data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.385592751Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"dataset is oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.396501277Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"snapshot is oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f@960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.407151329Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fc00f490-fb94-48aa-87e9-90293e6de49d/crucible/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f\", \"960a2b67-7152-477d-a1d3-11128f024a61\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.40722943Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e1a4adc6-e1d5-46d3-bb41-f127f8260d3f-960a2b67-7152-477d-a1d3-11128f024a61 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.421487764Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}
{"msg":"applying SMF actions for region e1a4adc6-e1d5-46d3-bb41-f127f8260d3f running snapshot 960a2b67-7152-477d-a1d3-11128f024a61 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.421690098Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.421759729Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"uri":"/crucible/0/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f/snapshots/960a2b67-7152-477d-a1d3-11128f024a61/run","method":"POST","req_id":"6c4e15e3-da79-4e3d-b01d-3aa8f985c917","remote_addr":"[fd00:1122:3344:101::a]:63832","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":36188,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f-960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.425440743Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f-960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.430423442Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e1a4adc6-e1d5-46d3-bb41-f127f8260d3f-960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.443716306Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e1a4adc6-e1d5-46d3-bb41-f127f8260d3f/.zfs/snapshot/960a2b67-7152-477d-a1d3-11128f024a61","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.444009642Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.444069033Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.444115294Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.444157304Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.444214386Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.450138254Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.45650303Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"worker"}
{"msg":"region e1a4adc6-e1d5-46d3-bb41-f127f8260d3f running snapshot 960a2b67-7152-477d-a1d3-11128f024a61 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-15T20:00:00.45652024Z","hostname":"oxz_crucible_b51dd4ca-27b7-4b94-b320-991bb51f99f0","pid":22540,"component":"datafile"}