[ Jun 28 20:20:18 Enabled. ]
[ Jun 28 20:20:18 Rereading configuration. ]
[ Jun 28 20:20:20 Rereading configuration. ]
[ Jun 28 20:20:23 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:20:23.974176502Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:20:23.977729864Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:20:23.977748414Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:20:23.997641076Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:20:24.112688838Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.696486625Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:47334"}
{"msg":"region e329a71e-292d-450f-962e-04d8b14e5318 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.705066336Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.710974607Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"676c99c0-1ea1-4f73-931c-c7a244e52094","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11156,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.711001987Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.721471285Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.721508245Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e329a71e-292d-450f-962e-04d8b14e5318\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e329a71e-292d-450f-962e-04d8b14e5318\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.778508899Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"e329a71e-292d-450f-962e-04d8b14e5318","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e329a71e-292d-450f-962e-04d8b14e5318\" \"--data\" \"/data/regions/e329a71e-292d-450f-962e-04d8b14e5318\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.77858957Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"e329a71e-292d-450f-962e-04d8b14e5318","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.892092747Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"e329a71e-292d-450f-962e-04d8b14e5318","component":"worker"}
{"msg":"region e329a71e-292d-450f-962e-04d8b14e5318 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.892141947Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.892222118Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.892626409Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.896133352Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.906141388Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.909811001Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.910405843Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.912775611Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.912823341Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.912859642Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.920474249Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:13.927101703Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.048242947Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"dc3fc71b-45c0-45f2-8d01-531fc0faa033","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.548127941Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/e329a71e-292d-450f-962e-04d8b14e5318","method":"GET","req_id":"7860ea99-f9f7-4978-9689-3e811365a932","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"path is \"/data/regions/e329a71e-292d-450f-962e-04d8b14e5318\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.550146658Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/e329a71e-292d-450f-962e-04d8b14e5318","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.560619336Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"region e329a71e-292d-450f-962e-04d8b14e5318 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.570325521Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.570476411Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.570557781Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/e329a71e-292d-450f-962e-04d8b14e5318","method":"DELETE","req_id":"b177340e-72d6-46a3-956a-8cf3f709b9cf","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20460,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e329a71e-292d-450f-962e-04d8b14e5318 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.571563055Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.57283258Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/e329a71e-292d-450f-962e-04d8b14e5318","method":"GET","req_id":"3b208eb4-4472-47cc-a053-4c1e1d91dc8e","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.57570611Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/e329a71e-292d-450f-962e-04d8b14e5318\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.585491405Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"e329a71e-292d-450f-962e-04d8b14e5318","component":"worker"}
{"msg":"region e329a71e-292d-450f-962e-04d8b14e5318 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.63972712Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:14.65084506Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/e329a71e-292d-450f-962e-04d8b14e5318","method":"GET","req_id":"cb93d1f2-450c-4353-b312-56b4aef5241d","remote_addr":"[fd00:1122:3344:101::c]:47334","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":572,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.324040173Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48463"}
{"msg":"region d62722e3-8f34-470e-9573-767d29dd8041 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.324175194Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.324361375Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.324373275Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"86053f37-0ff6-49f3-b593-8d5563a0f03b","remote_addr":"[fd00:1122:3344:101::a]:48463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.334464691Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.334518421Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d62722e3-8f34-470e-9573-767d29dd8041\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d62722e3-8f34-470e-9573-767d29dd8041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.381345649Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"d62722e3-8f34-470e-9573-767d29dd8041","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d62722e3-8f34-470e-9573-767d29dd8041\" \"--data\" \"/data/regions/d62722e3-8f34-470e-9573-767d29dd8041\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.381407539Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"d62722e3-8f34-470e-9573-767d29dd8041","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.468941423Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"d62722e3-8f34-470e-9573-767d29dd8041","component":"worker"}
{"msg":"region d62722e3-8f34-470e-9573-767d29dd8041 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.468986534Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.469063824Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.470496319Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.475448657Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.484939711Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.490124329Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.4904216Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.49047568Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.490514441Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.490553071Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.494694406Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.501029249Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:21:35.693968911Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"a9a55991-4fc3-4ecd-b9a9-b74a6f8ddb3c","remote_addr":"[fd00:1122:3344:101::a]:48463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.816676604Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64610"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.817238936Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/d62722e3-8f34-470e-9573-767d29dd8041","method":"GET","req_id":"cb3df9c5-72bc-4894-8dad-7fc26d574616","remote_addr":"[fd00:1122:3344:101::a]:64610","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":537,"response_code":"200"}
{"msg":"path is \"/data/regions/d62722e3-8f34-470e-9573-767d29dd8041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.817402357Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.827590903Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"snapshot is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041@19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.837178448Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041\", \"19891c95-ff5a-4815-a720-829e6cd5a4f6\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.837213768Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.849940524Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/d62722e3-8f34-470e-9573-767d29dd8041/snapshots/19891c95-ff5a-4815-a720-829e6cd5a4f6","method":"GET","req_id":"ed732438-6eaa-47e3-ab4e-3c13bbe1bb48","remote_addr":"[fd00:1122:3344:101::a]:64610","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":32544,"response_code":"200"}
{"msg":"path is \"/data/regions/d62722e3-8f34-470e-9573-767d29dd8041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.850107984Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.858809645Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"snapshot is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041@19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.867520676Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/d62722e3-8f34-470e-9573-767d29dd8041\", \"19891c95-ff5a-4815-a720-829e6cd5a4f6\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.867550757Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot d62722e3-8f34-470e-9573-767d29dd8041-19891c95-ff5a-4815-a720-829e6cd5a4f6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.879823021Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions for region d62722e3-8f34-470e-9573-767d29dd8041 running snapshot 19891c95-ff5a-4815-a720-829e6cd5a4f6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.880483363Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.880558853Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/d62722e3-8f34-470e-9573-767d29dd8041/snapshots/19891c95-ff5a-4815-a720-829e6cd5a4f6/run","method":"POST","req_id":"7bdb1be8-3fe4-485d-b772-cbb592b98a64","remote_addr":"[fd00:1122:3344:101::a]:64610","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":30486,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-d62722e3-8f34-470e-9573-767d29dd8041-19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.887851769Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-d62722e3-8f34-470e-9573-767d29dd8041-19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.894658724Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-d62722e3-8f34-470e-9573-767d29dd8041-19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.906749757Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d62722e3-8f34-470e-9573-767d29dd8041/.zfs/snapshot/19891c95-ff5a-4815-a720-829e6cd5a4f6","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.907027838Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.907081749Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.907121609Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.907162099Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.907213309Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.910714342Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.915087257Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"region d62722e3-8f34-470e-9573-767d29dd8041 running snapshot 19891c95-ff5a-4815-a720-829e6cd5a4f6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:26:57.915095717Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.853936792Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50684"}
{"msg":"region 226e2e42-544b-4b87-b9d5-9eab3566ca9d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.854015652Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.854226193Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.854238103Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"c73f0cd0-dd43-48f8-969c-98c3e7ca5f3c","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":266,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.864464219Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.86451212Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"226e2e42-544b-4b87-b9d5-9eab3566ca9d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.924562375Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"226e2e42-544b-4b87-b9d5-9eab3566ca9d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"226e2e42-544b-4b87-b9d5-9eab3566ca9d\" \"--data\" \"/data/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.926553252Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"226e2e42-544b-4b87-b9d5-9eab3566ca9d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.995148978Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"226e2e42-544b-4b87-b9d5-9eab3566ca9d","component":"worker"}
{"msg":"region 226e2e42-544b-4b87-b9d5-9eab3566ca9d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.995192058Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.995303369Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.996281033Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:00.999226753Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.006266231Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.009011668Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.009286029Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.009345409Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.009385589Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.00943758Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.013039753Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.024772095Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:01.106162326Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"9fa677df-99b8-435f-9391-e90fca9e18a7","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.214042792Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","method":"GET","req_id":"2b08bf8e-3464-4f5d-b71d-2b63ef47bb26","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.216973983Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.231939766Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"region 226e2e42-544b-4b87-b9d5-9eab3566ca9d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.240504217Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.240783908Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.240835598Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","method":"DELETE","req_id":"63edddbd-d77c-4fe8-a7fb-e492bd81e9c7","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23888,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-226e2e42-544b-4b87-b9d5-9eab3566ca9d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.242416924Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.243273957Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","method":"GET","req_id":"9ef38d43-5a82-4914-a1d1-77ebfe369648","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.248883937Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6adfb7f2-0d81-42e0-a47c-52d4c5bc9b11/crucible/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.258646452Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"region":"226e2e42-544b-4b87-b9d5-9eab3566ca9d","component":"worker"}
{"msg":"region 226e2e42-544b-4b87-b9d5-9eab3566ca9d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.321479448Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-28T20:27:26.344129639Z","hostname":"oxz_crucible_a46ed8b8-c687-449d-94f8-3ba05af89482","pid":22033,"uri":"/crucible/0/regions/226e2e42-544b-4b87-b9d5-9eab3566ca9d","method":"GET","req_id":"10a2dcac-a0de-4d55-9e59-1e24da574312","remote_addr":"[fd00:1122:3344:101::a]:50684","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":599,"response_code":"200"}