{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.665643915Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59827"}
{"msg":"region 1e5d126a-ce37-4023-a4db-1513214ac03b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.68854562Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.69505664Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.698900333Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions","method":"POST","req_id":"ac910ba1-b41f-489b-becd-482d164cd9cd","remote_addr":"[fd00:1122:3344:101::a]:59827","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17042,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.70752289Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.70753753Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1e5d126a-ce37-4023-a4db-1513214ac03b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.760663045Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"region":"1e5d126a-ce37-4023-a4db-1513214ac03b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1e5d126a-ce37-4023-a4db-1513214ac03b\" \"--data\" \"/data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.762524355Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"region":"1e5d126a-ce37-4023-a4db-1513214ac03b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:22.937788311Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions","method":"POST","req_id":"bc3e0ed3-10cf-45b5-bbe9-688be569e485","remote_addr":"[fd00:1122:3344:101::a]:59827","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.019433201Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"region":"1e5d126a-ce37-4023-a4db-1513214ac03b","component":"worker"}
{"msg":"region 1e5d126a-ce37-4023-a4db-1513214ac03b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.019480773Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.019571996Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.019988589Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.02529971Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.041291593Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.044356052Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.05209515Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.054006691Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.054083264Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.054134795Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.057471593Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.061982467Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:30:23.497016621Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions","method":"POST","req_id":"d6a43850-8e73-4ca3-90d7-9ac0408bd5a6","remote_addr":"[fd00:1122:3344:101::a]:59827","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.008961387Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44959"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.00906074Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","method":"GET","req_id":"4b9d472c-b381-4a75-be21-84b7bdae7daa","remote_addr":"[fd00:1122:3344:101::a]:44959","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"path is \"/data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.012317585Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"dataset is oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.022376218Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"snapshot is oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b@be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.030809298Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\", \"be3f759b-311c-4622-b8b1-9bfd2a58ea57\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.030831499Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.042151122Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions/1e5d126a-ce37-4023-a4db-1513214ac03b/snapshots/be3f759b-311c-4622-b8b1-9bfd2a58ea57","method":"GET","req_id":"5117da44-5683-4533-970d-5ff878b9a32e","remote_addr":"[fd00:1122:3344:101::a]:44959","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":32900,"response_code":"200"}
{"msg":"path is \"/data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.04238322Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"dataset is oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.050098727Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"snapshot is oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b@be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.059318723Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0598fc08-1617-40dc-ac47-23572069cd01/crucible/regions/1e5d126a-ce37-4023-a4db-1513214ac03b\", \"be3f759b-311c-4622-b8b1-9bfd2a58ea57\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.059363065Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 1e5d126a-ce37-4023-a4db-1513214ac03b-be3f759b-311c-4622-b8b1-9bfd2a58ea57 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.067920499Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}
{"msg":"applying SMF actions for region 1e5d126a-ce37-4023-a4db-1513214ac03b running snapshot be3f759b-311c-4622-b8b1-9bfd2a58ea57 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.068576681Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.068604772Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"uri":"/crucible/0/regions/1e5d126a-ce37-4023-a4db-1513214ac03b/snapshots/be3f759b-311c-4622-b8b1-9bfd2a58ea57/run","method":"POST","req_id":"c2308d5b-2281-46ef-9b5a-90f75b949cc2","remote_addr":"[fd00:1122:3344:101::a]:44959","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26275,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076542976Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076630729Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076711722Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076718832Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076762543Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076812195Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076914848Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.076993091Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.077042892Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.077093254Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.091211767Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-1e5d126a-ce37-4023-a4db-1513214ac03b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.091681052Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1e5d126a-ce37-4023-a4db-1513214ac03b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.10187842Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1e5d126a-ce37-4023-a4db-1513214ac03b-be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118424231Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e5d126a-ce37-4023-a4db-1513214ac03b/.zfs/snapshot/be3f759b-311c-4622-b8b1-9bfd2a58ea57","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118649348Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118703109Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118743711Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118782082Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.118832024Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.124592379Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.129585249Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"worker"}
{"msg":"region 1e5d126a-ce37-4023-a4db-1513214ac03b running snapshot be3f759b-311c-4622-b8b1-9bfd2a58ea57 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-06T08:35:46.12960315Z","hostname":"oxz_crucible_04970943-974c-452d-8399-e1adef41e3ec","pid":21434,"component":"datafile"}