[ Jun 25 04:50:02 Enabled. ]
[ Jun 25 04:50:02 Rereading configuration. ]
[ Jun 25 04:50:04 Rereading configuration. ]
[ Jun 25 04:50:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.116396077Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.120066705Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.120085566Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.142636686Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.237630343Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.935935546Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34860"}
{"msg":"region a6b54d89-7b3d-47f4-b533-a3cb4ee64863 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.949072136Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.953360474Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"87a2278a-fbce-48d7-81c4-a93b6f7df9f4","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":6439,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.953379234Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.964051055Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.964083305Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a6b54d89-7b3d-47f4-b533-a3cb4ee64863\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.022979456Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"a6b54d89-7b3d-47f4-b533-a3cb4ee64863","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a6b54d89-7b3d-47f4-b533-a3cb4ee64863\" \"--data\" \"/data/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.024846495Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"a6b54d89-7b3d-47f4-b533-a3cb4ee64863","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.142678437Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"a6b54d89-7b3d-47f4-b533-a3cb4ee64863","component":"worker"}
{"msg":"region a6b54d89-7b3d-47f4-b533-a3cb4ee64863 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.142726578Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.142831379Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.143296787Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.147694277Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.155679975Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.160561222Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.166147462Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.168196215Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.168273386Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.168332757Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.172803198Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.176157531Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.209520544Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"4cdc7124-7911-4649-85f5-c4c49defafdf","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":717,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.466078661Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","method":"GET","req_id":"b1c30845-1a32-4f77-8d81-bf2f9fe5ef4f","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"path is \"/data/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.46918109Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"dataset is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.478458598Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"region a6b54d89-7b3d-47f4-b533-a3cb4ee64863 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.487168258Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.48730562Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-a6b54d89-7b3d-47f4-b533-a3cb4ee64863 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.488343206Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.489318852Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","method":"DELETE","req_id":"74c900de-f124-4b37-a926-6a1312b8804b","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20612,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.491276643Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","method":"GET","req_id":"de147876-b282-4f8d-8146-04b70d84d25b","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.495153245Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.504334092Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"a6b54d89-7b3d-47f4-b533-a3cb4ee64863","component":"worker"}
{"msg":"region a6b54d89-7b3d-47f4-b533-a3cb4ee64863 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.56368967Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.619149995Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/a6b54d89-7b3d-47f4-b533-a3cb4ee64863","method":"GET","req_id":"8b33b9ea-1a6d-448e-bcca-ef14e7f3cd61","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"region e523ba1f-c739-4990-9403-0b0de49a0802 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.213777217Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.213920219Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.213929949Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"d7d1435b-56da-4e75-82db-bc62089ce446","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":215,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.224465567Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.224503848Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e523ba1f-c739-4990-9403-0b0de49a0802\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e523ba1f-c739-4990-9403-0b0de49a0802\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.289510766Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"e523ba1f-c739-4990-9403-0b0de49a0802","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e523ba1f-c739-4990-9403-0b0de49a0802\" \"--data\" \"/data/regions/e523ba1f-c739-4990-9403-0b0de49a0802\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.289566256Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"e523ba1f-c739-4990-9403-0b0de49a0802","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.35990003Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"e523ba1f-c739-4990-9403-0b0de49a0802","component":"worker"}
{"msg":"region e523ba1f-c739-4990-9403-0b0de49a0802 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.359959941Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.360060002Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.361480775Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.364523213Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.372808566Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.37554884Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.375775123Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.37622735Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.376240951Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.376246381Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.37935627Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.383319573Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.574955813Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"a38ee76d-5b88-4df9-b8d9-0b4e2456241a","remote_addr":"[fd00:1122:3344:101::a]:34860","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.673090096Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36033"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.673237919Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/e523ba1f-c739-4990-9403-0b0de49a0802","method":"GET","req_id":"014dcde4-77e7-46df-9c2d-833adc078dd0","remote_addr":"[fd00:1122:3344:101::a]:36033","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"path is \"/data/regions/e523ba1f-c739-4990-9403-0b0de49a0802\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.676437639Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"dataset is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.685733868Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"snapshot is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802@d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.695043787Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802\", \"d7341147-ce54-48ba-ab4a-2579a151f90d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.695073677Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.708338449Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/e523ba1f-c739-4990-9403-0b0de49a0802/snapshots/d7341147-ce54-48ba-ab4a-2579a151f90d","method":"GET","req_id":"ba387143-905b-4544-913d-568ce6edcaad","remote_addr":"[fd00:1122:3344:101::a]:36033","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":35037,"response_code":"200"}
{"msg":"path is \"/data/regions/e523ba1f-c739-4990-9403-0b0de49a0802\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.708492702Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"dataset is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.717544346Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"snapshot is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802@d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.725895779Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/e523ba1f-c739-4990-9403-0b0de49a0802\", \"d7341147-ce54-48ba-ab4a-2579a151f90d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.72592631Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e523ba1f-c739-4990-9403-0b0de49a0802-d7341147-ce54-48ba-ab4a-2579a151f90d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.734957954Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions for region e523ba1f-c739-4990-9403-0b0de49a0802 running snapshot d7341147-ce54-48ba-ab4a-2579a151f90d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.735877029Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.73593596Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/e523ba1f-c739-4990-9403-0b0de49a0802/snapshots/d7341147-ce54-48ba-ab4a-2579a151f90d/run","method":"POST","req_id":"f4c2095a-e3cc-4d6d-9fcb-ede0299434d7","remote_addr":"[fd00:1122:3344:101::a]:36033","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27473,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-e523ba1f-c739-4990-9403-0b0de49a0802-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.740810267Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e523ba1f-c739-4990-9403-0b0de49a0802-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.74723089Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e523ba1f-c739-4990-9403-0b0de49a0802-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.758137094Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e523ba1f-c739-4990-9403-0b0de49a0802/.zfs/snapshot/d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.758445319Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.75849635Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.758535161Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.758578381Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.758621982Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.762033916Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.76602218Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"region e523ba1f-c739-4990-9403-0b0de49a0802 running snapshot d7341147-ce54-48ba-ab4a-2579a151f90d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.76603382Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505001794Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52872"}
{"msg":"region 201fe9ba-08c0-4a7d-a49c-f49c4ba114cc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505273088Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505432141Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505445961Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"eeb4b52a-a49c-4788-8fa9-8121accbd0fb","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":201,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.515528652Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.515555332Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.574502233Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\" \"--data\" \"/data/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.574556474Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.640324514Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","component":"worker"}
{"msg":"region 201fe9ba-08c0-4a7d-a49c-f49c4ba114cc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.640385195Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.640476146Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.641535494Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.644050954Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.651039295Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.654134865Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.654340108Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.654406159Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.65444988Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.65448911Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.657550189Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.667215653Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.825166126Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions","method":"POST","req_id":"6c877fd7-13af-4bbf-99ba-8b1af5e8a5be","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":640,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.151524481Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","method":"GET","req_id":"6ee0cc87-d797-476b-9d75-0ad57ec63f96","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"path is \"/data/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.15462712Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"dataset is oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.164689581Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"region 201fe9ba-08c0-4a7d-a49c-f49c4ba114cc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.173582253Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.173784546Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.173839447Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","method":"DELETE","req_id":"12e9567f-84f2-4ae0-b1da-17e0d7652e56","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19240,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-201fe9ba-08c0-4a7d-a49c-f49c4ba114cc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.175423352Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.17838448Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","method":"GET","req_id":"d50a2957-ca96-41a2-b033-0c6b52044101","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":680,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.181365157Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d11f1de3-731b-46f3-8259-abe2174edcbd/crucible/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.191765623Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"region":"201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","component":"worker"}
{"msg":"region 201fe9ba-08c0-4a7d-a49c-f49c4ba114cc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.261859082Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.26294678Z","hostname":"oxz_crucible_6511791c-64e1-4204-bec9-3e5138531ad9","pid":22813,"uri":"/crucible/0/regions/201fe9ba-08c0-4a7d-a49c-f49c4ba114cc","method":"GET","req_id":"5e01adbd-de02-4463-a7a7-08e1896f1cf9","remote_addr":"[fd00:1122:3344:101::a]:52872","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":"200"}