{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.955520327Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57430"}
{"msg":"region 58eb1c4f-75fb-47a0-8c23-a27b447494f8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.976780079Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.983563337Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.98740241Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"5f752a7b-14da-46df-91df-38b2984ccfc9","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16670,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.994934292Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:19.994956633Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"58eb1c4f-75fb-47a0-8c23-a27b447494f8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.061269661Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"58eb1c4f-75fb-47a0-8c23-a27b447494f8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"58eb1c4f-75fb-47a0-8c23-a27b447494f8\" \"--data\" \"/data/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.063230404Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"58eb1c4f-75fb-47a0-8c23-a27b447494f8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.138673956Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"8d821db0-9310-455a-9318-327e28bc60c4","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.23632622Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"58eb1c4f-75fb-47a0-8c23-a27b447494f8","component":"worker"}
{"msg":"region 58eb1c4f-75fb-47a0-8c23-a27b447494f8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.236373872Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.236488025Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.236965921Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.242089825Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.258921985Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.262026785Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.269290348Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.271323484Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.271379975Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.271421577Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.275430725Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.279336211Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:20.393579428Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"5388ee83-0679-49cd-af5c-57156f349c21","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.0734515Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","method":"GET","req_id":"2875d181-46cd-4d8d-ad1d-945a36637342","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"path is \"/data/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.075854147Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"dataset is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.085660942Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"region 58eb1c4f-75fb-47a0-8c23-a27b447494f8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.093960788Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.094138854Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.094206986Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","method":"DELETE","req_id":"60a4145a-694a-4b99-a26e-c640694e025c","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18385,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-58eb1c4f-75fb-47a0-8c23-a27b447494f8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.095389264Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.097777051Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","method":"GET","req_id":"fec09738-83fc-4801-ba3a-8caecec30ed1","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.099347461Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.109142285Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"58eb1c4f-75fb-47a0-8c23-a27b447494f8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.16412954Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","method":"GET","req_id":"e9c54b2e-1658-4314-b433-c8caa52d6b92","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"region 58eb1c4f-75fb-47a0-8c23-a27b447494f8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.169856114Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:21.242407903Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/58eb1c4f-75fb-47a0-8c23-a27b447494f8","method":"GET","req_id":"a5c8ad98-ea18-4d1b-96be-6e9ecdfd3b49","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"region f84bd0ac-0fda-47d7-8823-3b54d528e25b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.665652226Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.665834142Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"dd5ddd49-768d-4e63-998a-f1c84fa70055","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":260,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.665865172Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.675794841Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.675827042Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f84bd0ac-0fda-47d7-8823-3b54d528e25b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.726285482Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"f84bd0ac-0fda-47d7-8823-3b54d528e25b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f84bd0ac-0fda-47d7-8823-3b54d528e25b\" \"--data\" \"/data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.728176763Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"f84bd0ac-0fda-47d7-8823-3b54d528e25b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.795823474Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"f84bd0ac-0fda-47d7-8823-3b54d528e25b","component":"worker"}
{"msg":"region f84bd0ac-0fda-47d7-8823-3b54d528e25b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.795862845Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.795962088Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.797326112Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.800332519Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.80690718Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.809668908Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.809866325Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.809920896Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.809956697Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.809993799Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.813475041Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.817953174Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.828212634Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"d44f88be-d135-4240-991e-42397bd684d0","remote_addr":"[fd00:1122:3344:101::a]:57430","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.211792298Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51801"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.212023445Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","method":"GET","req_id":"5ccab3a1-f0a4-40fa-bd0d-a830a48680c8","remote_addr":"[fd00:1122:3344:101::a]:51801","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"path is \"/data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.214559906Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"dataset is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.223594007Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"snapshot is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b@be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.231624984Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\", \"be3f759b-311c-4622-b8b1-9bfd2a58ea57\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.231662755Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.242624747Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b/snapshots/be3f759b-311c-4622-b8b1-9bfd2a58ea57","method":"GET","req_id":"d7513bdf-3260-402a-b408-54904fab83ee","remote_addr":"[fd00:1122:3344:101::a]:51801","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":30541,"response_code":"200"}
{"msg":"path is \"/data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.242815363Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"dataset is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.252232636Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"snapshot is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b@be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.260522242Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b\", \"be3f759b-311c-4622-b8b1-9bfd2a58ea57\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.260557713Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f84bd0ac-0fda-47d7-8823-3b54d528e25b-be3f759b-311c-4622-b8b1-9bfd2a58ea57 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.271133912Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions for region f84bd0ac-0fda-47d7-8823-3b54d528e25b running snapshot be3f759b-311c-4622-b8b1-9bfd2a58ea57 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.271809854Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.271832075Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b/snapshots/be3f759b-311c-4622-b8b1-9bfd2a58ea57/run","method":"POST","req_id":"6703f7e6-7424-4b14-afdf-a3126101d3de","remote_addr":"[fd00:1122:3344:101::a]:51801","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":29047,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-f84bd0ac-0fda-47d7-8823-3b54d528e25b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.278427696Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f84bd0ac-0fda-47d7-8823-3b54d528e25b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.291175546Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f84bd0ac-0fda-47d7-8823-3b54d528e25b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.31374394Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f84bd0ac-0fda-47d7-8823-3b54d528e25b/.zfs/snapshot/be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.314296268Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.314408791Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.314473433Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.314527085Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.314596588Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.319900408Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.328578156Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"region f84bd0ac-0fda-47d7-8823-3b54d528e25b running snapshot be3f759b-311c-4622-b8b1-9bfd2a58ea57 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.328596887Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.982887983Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41599"}
{"msg":"region 879ea498-8e27-4945-8ba6-70ed60f8cded state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.983534124Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.98372656Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"aae1f540-9d2b-413f-98fd-9c1798cbebbc","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":808,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.983747651Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.994674271Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:48.994720353Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"879ea498-8e27-4945-8ba6-70ed60f8cded\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/879ea498-8e27-4945-8ba6-70ed60f8cded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.086680785Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"879ea498-8e27-4945-8ba6-70ed60f8cded","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"879ea498-8e27-4945-8ba6-70ed60f8cded\" \"--data\" \"/data/regions/879ea498-8e27-4945-8ba6-70ed60f8cded\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.088523034Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"879ea498-8e27-4945-8ba6-70ed60f8cded","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.201648785Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"879ea498-8e27-4945-8ba6-70ed60f8cded","component":"worker"}
{"msg":"region 879ea498-8e27-4945-8ba6-70ed60f8cded state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.201695017Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.201842941Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.20304811Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.208742103Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.224174718Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.229604792Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.229872911Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.229931123Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.229976034Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.230021966Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.236424191Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.251855036Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:49.337296669Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions","method":"POST","req_id":"9b88285b-98ad-4f4d-bf62-e12a7fd60623","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.091938223Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","method":"GET","req_id":"7df26a40-36bd-4293-8c45-48bb9ef7c113","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"path is \"/data/regions/879ea498-8e27-4945-8ba6-70ed60f8cded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.095252009Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"dataset is oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.105720555Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"region 879ea498-8e27-4945-8ba6-70ed60f8cded state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.113861086Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.114072003Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.114131265Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","method":"DELETE","req_id":"26f1e7fc-bc0d-430f-84d8-459647e9cb84","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18903,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-879ea498-8e27-4945-8ba6-70ed60f8cded (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.115742257Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.117552435Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","method":"GET","req_id":"123c7168-7a2e-464d-8bff-c83d1089eb60","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":512,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.126002906Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_b948a3f5-d7f5-4483-8269-3314ec240750/crucible/regions/879ea498-8e27-4945-8ba6-70ed60f8cded\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.136459932Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"region":"879ea498-8e27-4945-8ba6-70ed60f8cded","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.187715587Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","method":"GET","req_id":"c4c1662c-7f96-4f35-bc64-e6f3e0917741","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region 879ea498-8e27-4945-8ba6-70ed60f8cded state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.191252701Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:36:16.330556552Z","hostname":"oxz_crucible_6c49c9aa-017a-45e6-bf2d-9c667ffa913c","pid":21322,"uri":"/crucible/0/regions/879ea498-8e27-4945-8ba6-70ed60f8cded","method":"GET","req_id":"11938829-3662-42e2-82fd-2ea7229e6352","remote_addr":"[fd00:1122:3344:101::a]:41599","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32,"response_code":"200"}