{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.152336719Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46384"}
{"msg":"region ecb72ad0-1906-4152-94e9-9e72a25f467a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.15243449Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.153141723Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions","method":"POST","req_id":"291259b8-f33c-412d-bfa9-54ea435c899b","remote_addr":"[fd00:1122:3344:101::a]:46384","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":766,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.153156093Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.162837575Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.162864995Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ecb72ad0-1906-4152-94e9-9e72a25f467a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ecb72ad0-1906-4152-94e9-9e72a25f467a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.234860574Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"ecb72ad0-1906-4152-94e9-9e72a25f467a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ecb72ad0-1906-4152-94e9-9e72a25f467a\" \"--data\" \"/data/regions/ecb72ad0-1906-4152-94e9-9e72a25f467a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.236806639Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"ecb72ad0-1906-4152-94e9-9e72a25f467a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.301192303Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"ecb72ad0-1906-4152-94e9-9e72a25f467a","component":"worker"}
{"msg":"region ecb72ad0-1906-4152-94e9-9e72a25f467a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.301229144Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.301349556Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.301800614Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.306032049Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.320292782Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.323062581Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ecb72ad0-1906-4152-94e9-9e72a25f467a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.323378877Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.323434258Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.323476168Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.323516599Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.326846408Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.331852077Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.331936578Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.33202312Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.33203331Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332070031Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332118152Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332198313Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332253304Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332301145Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.332351016Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.341468608Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.341921286Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:21.348654366Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions","method":"POST","req_id":"ba6d22a0-1ffd-491c-aae1-bdc2bda000e2","remote_addr":"[fd00:1122:3344:101::a]:46384","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":682,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.708052704Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44464"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.710969806Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","method":"GET","req_id":"644f09c9-ed15-4d6b-9d96-6c7b01bdf0a8","remote_addr":"[fd00:1122:3344:101::a]:44464","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2888,"response_code":"200"}
{"msg":"path is \"/data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.711160969Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"dataset is oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.720237831Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"snapshot is oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb@96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.730021194Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb\", \"96b1f498-e4fa-43b0-89db-290a75012292\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.730071685Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.743697667Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb/snapshots/96b1f498-e4fa-43b0-89db-290a75012292","method":"GET","req_id":"f59a0133-783c-446f-ae02-3b1b511a06ee","remote_addr":"[fd00:1122:3344:101::a]:44464","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32564,"response_code":"200"}
{"msg":"path is \"/data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.743935121Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"dataset is oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.752437883Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"snapshot is oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb@96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.760522456Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb\", \"96b1f498-e4fa-43b0-89db-290a75012292\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.760558227Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ed87fbdc-c16c-4c5d-90cc-784636ca02cb-96b1f498-e4fa-43b0-89db-290a75012292 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.769624738Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"applying SMF actions for region ed87fbdc-c16c-4c5d-90cc-784636ca02cb running snapshot 96b1f498-e4fa-43b0-89db-290a75012292 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.769836212Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.769863922Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb/snapshots/96b1f498-e4fa-43b0-89db-290a75012292/run","method":"POST","req_id":"c591f18e-90b9-4303-a4ab-ed0e5734dccf","remote_addr":"[fd00:1122:3344:101::a]:44464","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25976,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771537452Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771618063Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771697335Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771704315Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771744466Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.771789656Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.772096732Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.772150643Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.772199323Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.772244644Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.775199127Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-ed87fbdc-c16c-4c5d-90cc-784636ca02cb-96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.775692396Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ed87fbdc-c16c-4c5d-90cc-784636ca02cb-96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.781934556Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ed87fbdc-c16c-4c5d-90cc-784636ca02cb-96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.790795394Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb/.zfs/snapshot/96b1f498-e4fa-43b0-89db-290a75012292","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.791007458Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.791064709Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.791098159Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.79114066Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.791192021Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.794743594Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.798487861Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"region ed87fbdc-c16c-4c5d-90cc-784636ca02cb running snapshot 96b1f498-e4fa-43b0-89db-290a75012292 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:22.798507431Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.324080195Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:32791"}
{"msg":"region b0312434-8481-417d-be57-763a8f4502ab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.324208207Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.324453972Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.324471302Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions","method":"POST","req_id":"6d04816f-2741-45a5-a649-9fa23b75d96c","remote_addr":"[fd00:1122:3344:101::a]:32791","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":334,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.334187685Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.334225435Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b0312434-8481-417d-be57-763a8f4502ab\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b0312434-8481-417d-be57-763a8f4502ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.407849423Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"b0312434-8481-417d-be57-763a8f4502ab","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b0312434-8481-417d-be57-763a8f4502ab\" \"--data\" \"/data/regions/b0312434-8481-417d-be57-763a8f4502ab\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.410004611Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"b0312434-8481-417d-be57-763a8f4502ab","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.476869379Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"b0312434-8481-417d-be57-763a8f4502ab","component":"worker"}
{"msg":"region b0312434-8481-417d-be57-763a8f4502ab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.47692396Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.477017932Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.47747367Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.480730838Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.487213903Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.491312186Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.49157607Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.491632421Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.491674652Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.491714543Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.495105193Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.50058723Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.500673102Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.500757333Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.500765093Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.500807684Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.500858305Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.504414118Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.504473079Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.5045238Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.504572391Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.508220816Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.510205891Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:25.513626982Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions","method":"POST","req_id":"973e20b3-9e49-41dd-bdcd-9f83f18f1457","remote_addr":"[fd00:1122:3344:101::a]:32791","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.775052268Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58509"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.775123349Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/b0312434-8481-417d-be57-763a8f4502ab","method":"GET","req_id":"098dac52-e244-4fe7-bf6e-087a7252d70f","remote_addr":"[fd00:1122:3344:101::a]:58509","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"path is \"/data/regions/b0312434-8481-417d-be57-763a8f4502ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.775253041Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"dataset is oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/b0312434-8481-417d-be57-763a8f4502ab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.786067084Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"region b0312434-8481-417d-be57-763a8f4502ab state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.793800931Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.794010445Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.796429028Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/b0312434-8481-417d-be57-763a8f4502ab","method":"DELETE","req_id":"1c06b176-4f47-4a91-9b9b-ec1aa59eeb43","remote_addr":"[fd00:1122:3344:101::a]:58509","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21214,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.79658833Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/b0312434-8481-417d-be57-763a8f4502ab","method":"GET","req_id":"36eea0c2-ddef-43c9-91e6-db8233338ab9","remote_addr":"[fd00:1122:3344:101::a]:58509","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-b0312434-8481-417d-be57-763a8f4502ab (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.797320173Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805132062Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805237894Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805336156Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805357676Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805410457Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805465808Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed87fbdc-c16c-4c5d-90cc-784636ca02cb","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805798994Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805860805Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805909326Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.805955537Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.809421548Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.810693101Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crucible/regions/b0312434-8481-417d-be57-763a8f4502ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.820451774Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"region":"b0312434-8481-417d-be57-763a8f4502ab","component":"worker"}
{"msg":"region b0312434-8481-417d-be57-763a8f4502ab state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.874694098Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:16:48.946829639Z","hostname":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","pid":22340,"uri":"/crucible/0/regions/b0312434-8481-417d-be57-763a8f4502ab","method":"GET","req_id":"c7531255-9a90-4fec-94e9-a640ee88f333","remote_addr":"[fd00:1122:3344:101::a]:58509","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":943,"response_code":"200"}