{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.853560497Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49145"}
{"msg":"region ce972d76-4fe7-4d4b-b46e-7928b1bb870a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.912458187Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.913202853Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.913214124Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions","method":"POST","req_id":"d799fe29-b2d3-4989-b238-bcd72c934b04","remote_addr":"[fd00:1122:3344:101::a]:49145","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":743,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.922354424Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.922378434Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ce972d76-4fe7-4d4b-b46e-7928b1bb870a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ce972d76-4fe7-4d4b-b46e-7928b1bb870a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:54.975447214Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"region":"ce972d76-4fe7-4d4b-b46e-7928b1bb870a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.040186018Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"region":"ce972d76-4fe7-4d4b-b46e-7928b1bb870a","component":"worker"}
{"msg":"region ce972d76-4fe7-4d4b-b46e-7928b1bb870a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.040242908Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.040337089Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.04420066Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044281872Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044364323Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044372093Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044410143Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044458284Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044548035Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044614836Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044660886Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.044707887Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.052558631Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.052951476Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.056093757Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.062335179Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.065228968Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ce972d76-4fe7-4d4b-b46e-7928b1bb870a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.065507791Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.065580042Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.065627783Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.065668694Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.069060518Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.073293564Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.191056557Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions","method":"POST","req_id":"51f7fc67-70f5-446f-9be2-1c1bbc7ad801","remote_addr":"[fd00:1122:3344:101::a]:49145","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.726350864Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56171"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.72906463Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","method":"GET","req_id":"1bb314ea-c24f-4da3-a4e7-86f942f26021","remote_addr":"[fd00:1122:3344:101::a]:56171","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2693,"response_code":"200"}
{"msg":"path is \"/data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.729224312Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.737812095Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0@767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.745652638Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0\", \"767f3fa6-de57-4fd6-ad9a-678ff91528ca\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.745678889Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.755753881Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0/snapshots/767f3fa6-de57-4fd6-ad9a-678ff91528ca","method":"GET","req_id":"ce0a451d-5e2b-4e4f-b3d4-f1a21c1fdc71","remote_addr":"[fd00:1122:3344:101::a]:56171","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26542,"response_code":"200"}
{"msg":"path is \"/data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.755895583Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.763886129Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0@767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.771899244Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0\", \"767f3fa6-de57-4fd6-ad9a-678ff91528ca\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.771937455Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 3afaa2d5-82c5-498f-9558-c2092959d7d0-767f3fa6-de57-4fd6-ad9a-678ff91528ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.780717691Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"applying SMF actions for region 3afaa2d5-82c5-498f-9558-c2092959d7d0 running snapshot 767f3fa6-de57-4fd6-ad9a-678ff91528ca (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.780859243Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.780892383Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0/snapshots/767f3fa6-de57-4fd6-ad9a-678ff91528ca/run","method":"POST","req_id":"7040c321-6711-48fc-a8db-2d97945f52e6","remote_addr":"[fd00:1122:3344:101::a]:56171","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25022,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781513901Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781621273Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781725274Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781734304Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781786525Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.781844165Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.78219743Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.782254841Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.782300771Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.782345432Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.78594878Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-3afaa2d5-82c5-498f-9558-c2092959d7d0-767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.787574211Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3afaa2d5-82c5-498f-9558-c2092959d7d0-767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.795463675Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3afaa2d5-82c5-498f-9558-c2092959d7d0-767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.804467844Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0/.zfs/snapshot/767f3fa6-de57-4fd6-ad9a-678ff91528ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.804752058Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.804809768Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.804850989Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.804890039Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.80494164Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.808128092Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.812861434Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"region 3afaa2d5-82c5-498f-9558-c2092959d7d0 running snapshot 767f3fa6-de57-4fd6-ad9a-678ff91528ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:55.812877045Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.858812874Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33669"}
{"msg":"region 0e740589-d6ba-4995-b400-87869f9a9b82 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.917478937Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.91768818Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.91770729Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions","method":"POST","req_id":"ee439e39-0586-43ae-8b80-986cd7ab246e","remote_addr":"[fd00:1122:3344:101::a]:33669","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":278,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.927158625Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:03:59.927182735Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0e740589-d6ba-4995-b400-87869f9a9b82\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/0e740589-d6ba-4995-b400-87869f9a9b82\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.119033895Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"region":"0e740589-d6ba-4995-b400-87869f9a9b82","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.123148049Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions","method":"POST","req_id":"26fd22b2-3be1-483e-a77e-b6e2c26800dd","remote_addr":"[fd00:1122:3344:101::a]:33669","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.526793291Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"region":"0e740589-d6ba-4995-b400-87869f9a9b82","component":"worker"}
{"msg":"region 0e740589-d6ba-4995-b400-87869f9a9b82 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.526840101Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.527060184Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.527578591Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.570828381Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.60795327Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions","method":"POST","req_id":"22b440c7-cdcf-459d-b2c6-9eb2127664ee","remote_addr":"[fd00:1122:3344:101::a]:33669","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.651486304Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.665919447Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.666297358Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.666340578Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.666379737Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.666415457Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.690821833Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702287364Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702399686Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702490017Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702504247Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702537977Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.702582788Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.752199642Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.752254173Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.752302724Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.752353914Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.793598537Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:00.796636377Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.721778809Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63817"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.722514858Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/0e740589-d6ba-4995-b400-87869f9a9b82","method":"GET","req_id":"8b3e5448-3870-4cb4-87a6-d1ddcc9733a1","remote_addr":"[fd00:1122:3344:101::a]:63817","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":711,"response_code":"200"}
{"msg":"path is \"/data/regions/0e740589-d6ba-4995-b400-87869f9a9b82\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.72265884Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/0e740589-d6ba-4995-b400-87869f9a9b82","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.732586681Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"region 0e740589-d6ba-4995-b400-87869f9a9b82 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.743332533Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.743491405Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.743538566Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/0e740589-d6ba-4995-b400-87869f9a9b82","method":"DELETE","req_id":"d25d4f1e-415d-4182-8dc2-25b67a1a79ba","remote_addr":"[fd00:1122:3344:101::a]:63817","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20902,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.743704868Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/0e740589-d6ba-4995-b400-87869f9a9b82","method":"GET","req_id":"240d6bb6-f27a-4afe-9bd0-2300e9c78885","remote_addr":"[fd00:1122:3344:101::a]:63817","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-0e740589-d6ba-4995-b400-87869f9a9b82 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.74462316Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755210309Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755321601Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755418952Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755427712Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755481143Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755535444Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3afaa2d5-82c5-498f-9558-c2092959d7d0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755870628Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.755930089Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.75597741Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.75602139Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.763130433Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.766558138Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/0e740589-d6ba-4995-b400-87869f9a9b82\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.774758296Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"region":"0e740589-d6ba-4995-b400-87869f9a9b82","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.79702187Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/0e740589-d6ba-4995-b400-87869f9a9b82","method":"GET","req_id":"730d7f16-426b-48bb-b119-c157b6f11cc5","remote_addr":"[fd00:1122:3344:101::a]:63817","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region 0e740589-d6ba-4995-b400-87869f9a9b82 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.833711684Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-24T05:04:41.941271282Z","hostname":"oxz_crucible_d063fe42-c3e4-4bc3-9c7a-ac021519333e","pid":23361,"uri":"/crucible/0/regions/0e740589-d6ba-4995-b400-87869f9a9b82","method":"GET","req_id":"ed71664a-23c4-47e6-be46-b56d21328fc3","remote_addr":"[fd00:1122:3344:101::a]:63817","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":39,"response_code":"200"}