{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.087521495Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43389"} {"msg":"region c63cc203-841d-4874-93f9-f10846a873be state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.136934634Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.142097089Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.147250181Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions","method":"POST","req_id":"b5871af5-382c-4c29-a3a4-1214ef508182","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17564,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.154902761Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.154925461Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c63cc203-841d-4874-93f9-f10846a873be\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c63cc203-841d-4874-93f9-f10846a873be\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.235757051Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"c63cc203-841d-4874-93f9-f10846a873be","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c63cc203-841d-4874-93f9-f10846a873be\" \"--data\" \"/data/regions/c63cc203-841d-4874-93f9-f10846a873be\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.238123312Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"c63cc203-841d-4874-93f9-f10846a873be","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.284294617Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions","method":"POST","req_id":"893a2850-83a3-436a-a567-a0bd9004593c","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.534899527Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"c63cc203-841d-4874-93f9-f10846a873be","component":"worker"} {"msg":"region c63cc203-841d-4874-93f9-f10846a873be state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.534951128Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.535055681Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.535502353Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.548873911Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.569829608Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.57295136Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c63cc203-841d-4874-93f9-f10846a873be","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.573661338Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.573741611Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.573779502Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.573820443Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.577886919Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.58213234Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:03.793024003Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions","method":"POST","req_id":"c0d75b29-2b27-416d-9cee-2590ee6070e1","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"region 1c88135b-4ddf-4915-bab2-fa23df6a7219 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:08.943510399Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:08.943690134Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions","method":"POST","req_id":"e6ef679d-ca39-467e-bdf3-6e7ccf9f98f9","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":220,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:08.943709414Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:08.95313354Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:08.953160091Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"1c88135b-4ddf-4915-bab2-fa23df6a7219\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.009563793Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"1c88135b-4ddf-4915-bab2-fa23df6a7219","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1c88135b-4ddf-4915-bab2-fa23df6a7219\" \"--data\" \"/data/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.01138214Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"1c88135b-4ddf-4915-bab2-fa23df6a7219","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.088886803Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"1c88135b-4ddf-4915-bab2-fa23df6a7219","component":"worker"} {"msg":"region 1c88135b-4ddf-4915-bab2-fa23df6a7219 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.088924244Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.089016066Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.089443107Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.093073742Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.100579568Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.103452413Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.103770531Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.103825343Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.103870424Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.103911025Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.107635722Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.11560895Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:09.278797369Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions","method":"POST","req_id":"2d8c8ab4-fa70-4af4-b9c8-05252bbc6979","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":708,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.011917482Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","method":"GET","req_id":"e37b1561-70a9-4aa1-9377-0fb2deb4ac39","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":116,"response_code":"200"} {"msg":"path is \"/data/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.015913186Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"dataset is oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.027779425Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"region 1c88135b-4ddf-4915-bab2-fa23df6a7219 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.037978952Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.038175927Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.038236889Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","method":"DELETE","req_id":"01d6a19e-c984-4251-99f0-e3b4d9e58836","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22357,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-1c88135b-4ddf-4915-bab2-fa23df6a7219 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.039342057Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.041920354Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","method":"GET","req_id":"52760128-0b05-46a7-87aa-a88f4f756f33","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.045414776Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_fb2d454c-3596-42d0-8ff3-f8219d6e3f47/crucible/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.05592894Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"region":"1c88135b-4ddf-4915-bab2-fa23df6a7219","component":"worker"} {"msg":"region 1c88135b-4ddf-4915-bab2-fa23df6a7219 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.107241149Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-12T04:15:35.167537683Z","hostname":"oxz_crucible_c469e1af-57b9-4787-8577-1e177df6a4ab","pid":23312,"uri":"/crucible/0/regions/1c88135b-4ddf-4915-bab2-fa23df6a7219","method":"GET","req_id":"edee9104-4091-4aab-a5c0-c71a212844e4","remote_addr":"[fd00:1122:3344:101::a]:43389","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":"200"}