{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.326148795Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53147"}
{"msg":"region a4034713-3c54-41b4-b603-5a84d3f62b86 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.326241867Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.330105461Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.330116721Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions","method":"POST","req_id":"e48f238d-9d80-40df-a2c0-228954c423b5","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":3909,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.341663171Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.341699242Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a4034713-3c54-41b4-b603-5a84d3f62b86\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a4034713-3c54-41b4-b603-5a84d3f62b86\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.405592247Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"a4034713-3c54-41b4-b603-5a84d3f62b86","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a4034713-3c54-41b4-b603-5a84d3f62b86\" \"--data\" \"/data/regions/a4034713-3c54-41b4-b603-5a84d3f62b86\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.405643109Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"a4034713-3c54-41b4-b603-5a84d3f62b86","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.50478575Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"a4034713-3c54-41b4-b603-5a84d3f62b86","component":"worker"}
{"msg":"region a4034713-3c54-41b4-b603-5a84d3f62b86 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.504837292Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.504928104Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.512950769Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513046312Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513135274Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513142245Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513185466Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513250158Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513356831Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513421932Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513471183Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.513520165Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.520015029Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.52043062Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.523413761Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.533887982Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.53678678Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4034713-3c54-41b4-b603-5a84d3f62b86","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.537031016Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.537085477Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.537126278Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.537169Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.540483669Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.544757543Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:27.610420276Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions","method":"POST","req_id":"1aea83f9-b14a-45c8-bc14-d15cf6017d19","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":807,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.184906875Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","method":"GET","req_id":"c2095ef3-02f8-40eb-b676-a87d3a363bfb","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":4862,"response_code":"200"}
{"msg":"path is \"/data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.188107701Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"dataset is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.197093552Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"snapshot is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4@8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.204921042Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\", \"8d18c031-6495-4508-9fab-b0e279bb4e77\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.204954943Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.217691585Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/03488bda-6686-4e77-8ade-2ff539db3ed4/snapshots/8d18c031-6495-4508-9fab-b0e279bb4e77","method":"GET","req_id":"b0301116-fa9a-406a-b04f-e9a49116ea61","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":29585,"response_code":"200"}
{"msg":"path is \"/data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.221515528Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"dataset is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.228835444Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"snapshot is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4@8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.235537244Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\", \"8d18c031-6495-4508-9fab-b0e279bb4e77\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.235561575Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 03488bda-6686-4e77-8ade-2ff539db3ed4-8d18c031-6495-4508-9fab-b0e279bb4e77 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.244522795Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"applying SMF actions for region 03488bda-6686-4e77-8ade-2ff539db3ed4 running snapshot 8d18c031-6495-4508-9fab-b0e279bb4e77 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.244725831Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.244748401Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/03488bda-6686-4e77-8ade-2ff539db3ed4/snapshots/8d18c031-6495-4508-9fab-b0e279bb4e77/run","method":"POST","req_id":"8f9147e8-9b45-4a34-af43-2161bf4c17d6","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23270,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245265355Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245357458Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.24543733Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.24544719Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245484071Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245529062Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245886482Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245947704Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.245995555Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.246040746Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.249608872Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-03488bda-6686-4e77-8ade-2ff539db3ed4-8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.251056341Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-03488bda-6686-4e77-8ade-2ff539db3ed4-8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.259395685Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-03488bda-6686-4e77-8ade-2ff539db3ed4-8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269022973Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4/.zfs/snapshot/8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269263309Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269331401Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269381133Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269429404Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.269489876Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.273682228Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.277678395Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"region 03488bda-6686-4e77-8ade-2ff539db3ed4 running snapshot 8d18c031-6495-4508-9fab-b0e279bb4e77 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.277687065Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"path is \"/data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.508479921Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"dataset is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.521113351Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"snapshot is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4@8d18c031-6495-4508-9fab-b0e279bb4e77","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.53263635Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/03488bda-6686-4e77-8ade-2ff539db3ed4\", \"8d18c031-6495-4508-9fab-b0e279bb4e77\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.532691231Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:29.542514485Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/03488bda-6686-4e77-8ade-2ff539db3ed4/snapshots/8d18c031-6495-4508-9fab-b0e279bb4e77/run","method":"POST","req_id":"527303c7-3373-44dd-ac5f-e03957cc0797","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":34048,"response_code":"200"}
{"msg":"region 18275363-5572-45cf-aa50-0ed79b555a9e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.353592427Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.353843764Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.353853354Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions","method":"POST","req_id":"a831de2e-9a8d-4a7a-b7e1-ab5efe3ce16f","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":332,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.363300618Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.363344319Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"18275363-5572-45cf-aa50-0ed79b555a9e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/18275363-5572-45cf-aa50-0ed79b555a9e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.417987446Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"18275363-5572-45cf-aa50-0ed79b555a9e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"18275363-5572-45cf-aa50-0ed79b555a9e\" \"--data\" \"/data/regions/18275363-5572-45cf-aa50-0ed79b555a9e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.419829205Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"18275363-5572-45cf-aa50-0ed79b555a9e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.493217486Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"18275363-5572-45cf-aa50-0ed79b555a9e","component":"worker"}
{"msg":"region 18275363-5572-45cf-aa50-0ed79b555a9e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.493253776Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.493360209Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.493976166Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494060928Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494149521Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494178191Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494192752Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494266194Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494648404Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494714546Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494767177Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.494817649Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.498472577Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.498929389Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.501492018Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.508430724Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.511795154Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.51201702Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.512091912Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.512144024Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.512196205Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.515880414Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.525210014Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:33.590269091Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions","method":"POST","req_id":"dc30cd0a-4b2a-4b64-8bc5-30ba4042d118","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.133356334Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/18275363-5572-45cf-aa50-0ed79b555a9e","method":"GET","req_id":"fc7f4cef-8af7-4d5e-902f-50051ea392a9","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"path is \"/data/regions/18275363-5572-45cf-aa50-0ed79b555a9e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.136368085Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"dataset is oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/18275363-5572-45cf-aa50-0ed79b555a9e","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.148729247Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"region 18275363-5572-45cf-aa50-0ed79b555a9e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.159642009Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.159804234Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-18275363-5572-45cf-aa50-0ed79b555a9e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.160945724Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.162156687Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/18275363-5572-45cf-aa50-0ed79b555a9e","method":"DELETE","req_id":"937ca4f1-7da6-4db5-a573-4c075b0b272e","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25814,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.164994873Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/18275363-5572-45cf-aa50-0ed79b555a9e","method":"GET","req_id":"cb01e988-d850-4441-81fe-4a2ef3beff0d","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166144144Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166242757Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166339499Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.16634684Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166395551Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166448592Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03488bda-6686-4e77-8ade-2ff539db3ed4","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166776401Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166856063Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166917185Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.166991237Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.171241481Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.173320507Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_99311bbe-9041-4ee9-9e49-88d0748815df/crucible/regions/18275363-5572-45cf-aa50-0ed79b555a9e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.183800548Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"region":"18275363-5572-45cf-aa50-0ed79b555a9e","component":"worker"}
{"msg":"region 18275363-5572-45cf-aa50-0ed79b555a9e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.242920415Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-29T07:20:58.28705809Z","hostname":"oxz_crucible_3a8c380d-d789-4a34-a21c-d9a7010e3531","pid":22831,"uri":"/crucible/0/regions/18275363-5572-45cf-aa50-0ed79b555a9e","method":"GET","req_id":"dca9aa4e-7d72-4a64-a33c-8aafdea2dc27","remote_addr":"[fd00:1122:3344:101::a]:53147","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":"200"}