{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.191963146Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:48690"} {"msg":"region 61fd8967-8607-4840-b8da-c2c3daff715a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.23417195Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.247882755Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions","method":"POST","req_id":"572a614e-ee23-4f21-a628-2acb1785ce5f","remote_addr":"[fd00:1122:3344:101::c]:48690","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32769,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.253142229Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.267653981Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.267690742Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"61fd8967-8607-4840-b8da-c2c3daff715a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/61fd8967-8607-4840-b8da-c2c3daff715a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.339988179Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"61fd8967-8607-4840-b8da-c2c3daff715a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"61fd8967-8607-4840-b8da-c2c3daff715a\" \"--data\" \"/data/regions/61fd8967-8607-4840-b8da-c2c3daff715a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.34006495Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"61fd8967-8607-4840-b8da-c2c3daff715a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.440811337Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions","method":"POST","req_id":"b7ff2ce0-f0e2-415b-b31b-0545846907ff","remote_addr":"[fd00:1122:3344:101::c]:48690","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":89,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.69336923Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"61fd8967-8607-4840-b8da-c2c3daff715a","component":"worker"} {"msg":"region 61fd8967-8607-4840-b8da-c2c3daff715a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.69343679Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.693585302Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.694176087Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.704480933Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.727408456Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.730791024Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/61fd8967-8607-4840-b8da-c2c3daff715a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.733692309Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.733754189Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.73379468Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.73383201Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.736939076Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:51.754854377Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:52.007241888Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions","method":"POST","req_id":"03f72d66-c516-4854-b3b9-179983dfdf78","remote_addr":"[fd00:1122:3344:101::c]:48690","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":47,"response_code":"200"} {"msg":"region 82673181-2d78-4871-8f46-ad92529dd93a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.649455464Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.649626806Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.649639786Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions","method":"POST","req_id":"818accbd-3eea-4b3f-99fb-bde661ddcc68","remote_addr":"[fd00:1122:3344:101::c]:48690","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":218,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.660249495Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.660272895Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"82673181-2d78-4871-8f46-ad92529dd93a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/82673181-2d78-4871-8f46-ad92529dd93a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.7298238Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"82673181-2d78-4871-8f46-ad92529dd93a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"82673181-2d78-4871-8f46-ad92529dd93a\" \"--data\" \"/data/regions/82673181-2d78-4871-8f46-ad92529dd93a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.72988766Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"82673181-2d78-4871-8f46-ad92529dd93a","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.812565275Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"82673181-2d78-4871-8f46-ad92529dd93a","component":"worker"} {"msg":"region 82673181-2d78-4871-8f46-ad92529dd93a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.812609505Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.812708686Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.814482171Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.823654828Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.833056067Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.837170582Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.837503445Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.837553415Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.837593606Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.837628396Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.84164452Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.847359568Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:51:57.919559515Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions","method":"POST","req_id":"0bfaf3c1-dbc2-4c4e-a106-cf6dddab4d40","remote_addr":"[fd00:1122:3344:101::c]:48690","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.121851483Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37261"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.124067552Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions/82673181-2d78-4871-8f46-ad92529dd93a","method":"GET","req_id":"ae6570a8-eab0-4042-bfa2-cd3d671d4bf1","remote_addr":"[fd00:1122:3344:101::c]:37261","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2158,"response_code":"200"} {"msg":"path is \"/data/regions/82673181-2d78-4871-8f46-ad92529dd93a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.126723964Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"dataset is oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/82673181-2d78-4871-8f46-ad92529dd93a","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.139425411Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"region 82673181-2d78-4871-8f46-ad92529dd93a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.149328414Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.14999279Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-82673181-2d78-4871-8f46-ad92529dd93a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.15124311Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.151988326Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions/82673181-2d78-4871-8f46-ad92529dd93a","method":"DELETE","req_id":"7c8ab41e-fded-4b80-9b02-b81720fc9202","remote_addr":"[fd00:1122:3344:101::c]:37261","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25284,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.155282714Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions/82673181-2d78-4871-8f46-ad92529dd93a","method":"GET","req_id":"46b36184-777c-4d43-bdfd-9304352603f3","remote_addr":"[fd00:1122:3344:101::c]:37261","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.163958627Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_f05af1cf-0314-48c8-a85e-ef01721e5ad8/crucible/regions/82673181-2d78-4871-8f46-ad92529dd93a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.174697647Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"region":"82673181-2d78-4871-8f46-ad92529dd93a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.229038644Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions/82673181-2d78-4871-8f46-ad92529dd93a","method":"GET","req_id":"51048cf5-62f9-4794-8465-b9f64a599226","remote_addr":"[fd00:1122:3344:101::c]:37261","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"region 82673181-2d78-4871-8f46-ad92529dd93a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.243742238Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T19:52:30.411270716Z","hostname":"oxz_crucible_7ab13eb3-3603-4015-827c-bdccdc230b3f","pid":25830,"uri":"/crucible/0/regions/82673181-2d78-4871-8f46-ad92529dd93a","method":"GET","req_id":"cf2eca86-d4f0-4ec9-8c16-c42ef33f32e7","remote_addr":"[fd00:1122:3344:101::c]:37261","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26,"response_code":"200"}