{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.633478741Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64149"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.637446422Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/38200716-b816-4734-a1df-9a49eee9a8d5","method":"GET","req_id":"8c90df66-fb06-4112-a333-aae74ec78a42","remote_addr":"[fd00:1122:3344:101::a]:64149","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":3946,"response_code":"200"}
{"msg":"path is \"/data/regions/38200716-b816-4734-a1df-9a49eee9a8d5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.637630576Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.646566569Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5@cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.654489101Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5\", \"cee6c5e6-5159-4aa2-b516-144334d7aab6\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.654516702Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.665605089Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/38200716-b816-4734-a1df-9a49eee9a8d5/snapshots/cee6c5e6-5159-4aa2-b516-144334d7aab6","method":"GET","req_id":"04403e71-7c1f-41be-b436-2ab8223f97bc","remote_addr":"[fd00:1122:3344:101::a]:64149","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27992,"response_code":"200"}
{"msg":"path is \"/data/regions/38200716-b816-4734-a1df-9a49eee9a8d5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.665742092Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.67444746Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5@cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.682482644Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/38200716-b816-4734-a1df-9a49eee9a8d5\", \"cee6c5e6-5159-4aa2-b516-144334d7aab6\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.682513185Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 38200716-b816-4734-a1df-9a49eee9a8d5-cee6c5e6-5159-4aa2-b516-144334d7aab6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.690833985Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"applying SMF actions for region 38200716-b816-4734-a1df-9a49eee9a8d5 running snapshot cee6c5e6-5159-4aa2-b516-144334d7aab6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.691839646Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.691865996Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/38200716-b816-4734-a1df-9a49eee9a8d5/snapshots/cee6c5e6-5159-4aa2-b516-144334d7aab6/run","method":"POST","req_id":"3e7af781-58b8-4d0e-93bb-12714933fecb","remote_addr":"[fd00:1122:3344:101::a]:64149","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26152,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-38200716-b816-4734-a1df-9a49eee9a8d5-cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.698082203Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-38200716-b816-4734-a1df-9a49eee9a8d5-cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.702230408Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-38200716-b816-4734-a1df-9a49eee9a8d5-cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714340966Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/38200716-b816-4734-a1df-9a49eee9a8d5/.zfs/snapshot/cee6c5e6-5159-4aa2-b516-144334d7aab6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714626472Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714678123Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714722554Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714763025Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.714810646Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.718953631Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.723744669Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"region 38200716-b816-4734-a1df-9a49eee9a8d5 running snapshot cee6c5e6-5159-4aa2-b516-144334d7aab6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:19:57.723754029Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.325166787Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34729"}
{"msg":"region fc16260f-b6b4-4a16-9a38-9266a13d351c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.3253089Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.325575956Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.325591276Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions","method":"POST","req_id":"99b429bd-2838-421e-a8b7-088ad1973700","remote_addr":"[fd00:1122:3344:101::a]:34729","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":371,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.334884476Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.334914837Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fc16260f-b6b4-4a16-9a38-9266a13d351c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.405556383Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"region":"fc16260f-b6b4-4a16-9a38-9266a13d351c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.471490382Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions","method":"POST","req_id":"e4f00944-6a00-4224-97c0-fb161c4bb22b","remote_addr":"[fd00:1122:3344:101::a]:34729","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.473860911Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"region":"fc16260f-b6b4-4a16-9a38-9266a13d351c","component":"worker"}
{"msg":"region fc16260f-b6b4-4a16-9a38-9266a13d351c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.473881741Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.474020084Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.475237939Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.481861835Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.489643194Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.493474552Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.493735667Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.493784579Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.493831729Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.49387361Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.497814191Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:00.50361112Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:01.14947238Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions","method":"POST","req_id":"a3f1bc5f-05c5-4b64-b76a-e33d39fedfb9","remote_addr":"[fd00:1122:3344:101::a]:34729","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.337159507Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64654"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.337219178Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","method":"GET","req_id":"1096fed2-5384-4872-b5a3-9b9271a58938","remote_addr":"[fd00:1122:3344:101::a]:64654","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"path is \"/data/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.337389552Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.349593152Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"region fc16260f-b6b4-4a16-9a38-9266a13d351c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.359899523Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.360076046Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.360128017Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","method":"DELETE","req_id":"a8014aee-2910-4908-8631-15f2dda04a53","remote_addr":"[fd00:1122:3344:101::a]:64654","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22774,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.360803701Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","method":"GET","req_id":"5846d493-e3ee-4118-8c09-08f3a3a5f9f3","remote_addr":"[fd00:1122:3344:101::a]:64654","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":549,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-fc16260f-b6b4-4a16-9a38-9266a13d351c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.3612345Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.367365075Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.376037793Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"region":"fc16260f-b6b4-4a16-9a38-9266a13d351c","component":"worker"}
{"msg":"region fc16260f-b6b4-4a16-9a38-9266a13d351c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.422386182Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:20:31.490762581Z","hostname":"oxz_crucible_dc2ab691-e287-4137-ac74-ff6958f580d1","pid":24175,"uri":"/crucible/0/regions/fc16260f-b6b4-4a16-9a38-9266a13d351c","method":"GET","req_id":"173fa232-0cf8-48fc-b72b-d037252f2f56","remote_addr":"[fd00:1122:3344:101::a]:64654","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":663,"response_code":"200"}