{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.283882879Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37349"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.28393804Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c","method":"GET","req_id":"5b0c98d5-f13e-4f05-a9f7-1917f5c0ca83","remote_addr":"[fd00:1122:3344:101::a]:37349","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.284100273Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.292909126Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c@47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.301178719Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c\", \"47d087a7-8a4c-4266-bd5a-a250733581f7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.30120541Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.313561864Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c/snapshots/47d087a7-8a4c-4266-bd5a-a250733581f7","method":"GET","req_id":"8dfb8e64-ea1b-4d59-960f-7836ad0b56a5","remote_addr":"[fd00:1122:3344:101::a]:37349","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29463,"response_code":"200"}
{"msg":"path is \"/data/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.313775678Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.32255228Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c@47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.330996006Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c\", \"47d087a7-8a4c-4266-bd5a-a250733581f7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.331026107Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot eda73f40-12d8-43f8-bd3f-4ee43bc2011c-47d087a7-8a4c-4266-bd5a-a250733581f7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.339986342Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"applying SMF actions for region eda73f40-12d8-43f8-bd3f-4ee43bc2011c running snapshot 47d087a7-8a4c-4266-bd5a-a250733581f7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.340533562Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.340596113Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c/snapshots/47d087a7-8a4c-4266-bd5a-a250733581f7/run","method":"POST","req_id":"354dd306-9c60-4b77-9e87-d0427fe7534c","remote_addr":"[fd00:1122:3344:101::a]:37349","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26902,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-eda73f40-12d8-43f8-bd3f-4ee43bc2011c-47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.349327455Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-eda73f40-12d8-43f8-bd3f-4ee43bc2011c-47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.355850918Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-eda73f40-12d8-43f8-bd3f-4ee43bc2011c-47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.367681423Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eda73f40-12d8-43f8-bd3f-4ee43bc2011c/.zfs/snapshot/47d087a7-8a4c-4266-bd5a-a250733581f7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.367883037Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.367929327Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.367967528Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.368004299Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.368046569Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.371498919Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.375957837Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"region eda73f40-12d8-43f8-bd3f-4ee43bc2011c running snapshot 47d087a7-8a4c-4266-bd5a-a250733581f7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:57.375970137Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.862677539Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58649"}
{"msg":"region 8d1b1b6d-7f10-4ce6-833d-dc8142392fba state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.862779341Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.862971114Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions","method":"POST","req_id":"54b0cf4d-e02e-4549-8768-0951c6485475","remote_addr":"[fd00:1122:3344:101::a]:58649","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":238,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.862985864Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.872041071Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.872068482Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8d1b1b6d-7f10-4ce6-833d-dc8142392fba\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:19:59.927621326Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"region":"8d1b1b6d-7f10-4ce6-833d-dc8142392fba","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.002071687Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"region":"8d1b1b6d-7f10-4ce6-833d-dc8142392fba","component":"worker"}
{"msg":"region 8d1b1b6d-7f10-4ce6-833d-dc8142392fba state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.002121658Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.00224425Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.003472022Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.006558695Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.013817921Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.016854824Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.017051177Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.017100358Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.017153309Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.0171919Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.020300844Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.030103384Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:00.172714678Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions","method":"POST","req_id":"a7a824a4-93bc-434e-b1bd-b155fb9ef5ce","remote_addr":"[fd00:1122:3344:101::a]:58649","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.222026213Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42370"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.222105225Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","method":"GET","req_id":"5fd27673-22ef-424f-8cc8-2c4e3497e531","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.222248707Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.231771093Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"region 8d1b1b6d-7f10-4ce6-833d-dc8142392fba state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.239469166Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.23967247Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.239735961Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","method":"DELETE","req_id":"920aeb7d-98d2-450c-a5bf-1e923f65d7c2","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17507,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.239886943Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","method":"GET","req_id":"7db66247-90d9-4512-ad2c-45da83d1c651","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-8d1b1b6d-7f10-4ce6-833d-dc8142392fba (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.241532222Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.246732222Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.256375289Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"region":"8d1b1b6d-7f10-4ce6-833d-dc8142392fba","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.307887193Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","method":"GET","req_id":"93ff985e-6f3b-4c75-9d0f-77d0c23e474d","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":449,"response_code":"200"}
{"msg":"region 8d1b1b6d-7f10-4ce6-833d-dc8142392fba state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.322644539Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-28T08:20:25.414243508Z","hostname":"oxz_crucible_9542a533-67a6-4f23-9386-d5cd39778666","pid":23141,"uri":"/crucible/0/regions/8d1b1b6d-7f10-4ce6-833d-dc8142392fba","method":"GET","req_id":"036ca793-3e68-40d9-a221-095bb3b51073","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22,"response_code":"200"}