{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.392504314Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37704"} {"msg":"region 00fddabc-dfdb-4d61-8b13-2b48f5313375 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.455312387Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.455527019Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.455539099Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"39291b46-ea6e-495e-98cc-3396b6cc58e6","remote_addr":"[fd00:1122:3344:101::a]:37704","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":964,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.464263744Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.464295845Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"00fddabc-dfdb-4d61-8b13-2b48f5313375\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.523591347Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"00fddabc-dfdb-4d61-8b13-2b48f5313375","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.593582092Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"00fddabc-dfdb-4d61-8b13-2b48f5313375","component":"worker"} {"msg":"region 00fddabc-dfdb-4d61-8b13-2b48f5313375 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.593625272Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.593716313Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.595448238Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.598316513Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.60494226Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.607858186Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.608061007Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.608112498Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.608153098Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.608191549Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.611252245Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.614969737Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:15:30.631117337Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"53827352-cc95-4280-9c3a-038cfa6b9843","remote_addr":"[fd00:1122:3344:101::a]:37704","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":602,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.964966135Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40439"} {"msg":"region 0280e492-71a4-4c0b-b63d-80261c9a9ad8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.965092966Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.966010844Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.966025394Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"41e3d322-fb8d-40d3-9e76-3bf14284f9df","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1014,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.975764158Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:21.975785538Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"0280e492-71a4-4c0b-b63d-80261c9a9ad8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/0280e492-71a4-4c0b-b63d-80261c9a9ad8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.048130552Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"0280e492-71a4-4c0b-b63d-80261c9a9ad8","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.160738753Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"0280e492-71a4-4c0b-b63d-80261c9a9ad8","component":"worker"} {"msg":"region 0280e492-71a4-4c0b-b63d-80261c9a9ad8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.160783893Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.160893454Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.167838124Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.172929648Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.189301339Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.197074537Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0280e492-71a4-4c0b-b63d-80261c9a9ad8","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.197286388Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.197338509Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.197378719Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.197418389Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.207331775Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.21951938Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:22.289341382Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"3232dbb2-1a0a-4828-9ca3-6e786063542a","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.132590884Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39589"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.132648364Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","method":"GET","req_id":"d50e0c4e-db6b-444a-bb5c-19c409f4e190","remote_addr":"[fd00:1122:3344:101::a]:39589","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":51,"response_code":"200"} {"msg":"path is \"/data/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.137758778Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.146568344Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375@96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.155074527Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375\", \"96809e0e-e8a2-4037-99e3-ad24d31612db\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.155124708Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.165095114Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375/snapshots/96809e0e-e8a2-4037-99e3-ad24d31612db","method":"GET","req_id":"6532770c-4145-4e17-8f33-adb5a7be5d7b","remote_addr":"[fd00:1122:3344:101::a]:39589","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32269,"response_code":"200"} {"msg":"path is \"/data/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.165254755Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.173195563Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375@96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.180302335Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375\", \"96809e0e-e8a2-4037-99e3-ad24d31612db\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.180330685Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 00fddabc-dfdb-4d61-8b13-2b48f5313375-96809e0e-e8a2-4037-99e3-ad24d31612db state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.18784765Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"applying SMF actions for region 00fddabc-dfdb-4d61-8b13-2b48f5313375 running snapshot 96809e0e-e8a2-4037-99e3-ad24d31612db (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.187996461Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.188042142Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375/snapshots/96809e0e-e8a2-4037-99e3-ad24d31612db/run","method":"POST","req_id":"6a05e424-b67a-4dbe-8140-1f74aa5d84e5","remote_addr":"[fd00:1122:3344:101::a]:39589","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22823,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-00fddabc-dfdb-4d61-8b13-2b48f5313375-96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.190661874Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-00fddabc-dfdb-4d61-8b13-2b48f5313375-96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.19951377Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-00fddabc-dfdb-4d61-8b13-2b48f5313375-96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.222362027Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/00fddabc-dfdb-4d61-8b13-2b48f5313375/.zfs/snapshot/96809e0e-e8a2-4037-99e3-ad24d31612db","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.22264081Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.222703821Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.222753201Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.222804981Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.222864592Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.230192905Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.235234158Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"region 00fddabc-dfdb-4d61-8b13-2b48f5313375 running snapshot 96809e0e-e8a2-4037-99e3-ad24d31612db state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:23.235247169Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.955080941Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63845"} {"msg":"region 57ed67c2-08be-4fa7-8a5f-8db60fb56cc0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.985763646Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.985943457Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.985954897Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"52ad4430-89f8-438f-a15d-256e2559214c","remote_addr":"[fd00:1122:3344:101::a]:63845","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":235,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.995230997Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:25.995268138Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"57ed67c2-08be-4fa7-8a5f-8db60fb56cc0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.068216147Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.189368591Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","component":"worker"} {"msg":"region 57ed67c2-08be-4fa7-8a5f-8db60fb56cc0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.189428422Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.189501032Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.192151435Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.1997096Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.215408266Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.228389748Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.22864116Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.22869229Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.228732461Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.228771081Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.235488859Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.244984731Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:26.342484782Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions","method":"POST","req_id":"1c8c6079-0f80-4d59-8195-dcd9704a66b2","remote_addr":"[fd00:1122:3344:101::a]:63845","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.833178224Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50123"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.833322845Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","method":"GET","req_id":"6927d4b9-25c2-4cd9-8df6-d087e5120b60","remote_addr":"[fd00:1122:3344:101::a]:50123","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"path is \"/data/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.833518736Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.850897046Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"region 57ed67c2-08be-4fa7-8a5f-8db60fb56cc0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.859235718Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.85939965Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.85944942Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","method":"DELETE","req_id":"a6c56454-c9dd-4810-b88a-08826b17e8c5","remote_addr":"[fd00:1122:3344:101::a]:50123","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25972,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.859562471Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","method":"GET","req_id":"718be111-a562-4013-8b2b-9651be485668","remote_addr":"[fd00:1122:3344:101::a]:50123","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-57ed67c2-08be-4fa7-8a5f-8db60fb56cc0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.861114814Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.879432492Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.888274039Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"region":"57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","component":"worker"} {"msg":"region 57ed67c2-08be-4fa7-8a5f-8db60fb56cc0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.954384879Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-03T08:21:58.983040186Z","hostname":"oxz_crucible_b4cada8f-52e5-4a46-9672-57256c24f8d3","pid":21520,"uri":"/crucible/0/regions/57ed67c2-08be-4fa7-8a5f-8db60fb56cc0","method":"GET","req_id":"d91e4599-cb14-4771-b2d4-2923354c3fc0","remote_addr":"[fd00:1122:3344:101::a]:50123","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":128,"response_code":"200"}