{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.194052639Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42098"} {"msg":"region 056485ab-7f6e-4ae7-84bc-36f35bf53bb1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.243372672Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.247045433Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.247061763Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions","method":"POST","req_id":"8622a4c4-f2b9-42a8-b9f0-de443d029384","remote_addr":"[fd00:1122:3344:101::c]:42098","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20758,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.257366587Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.257385387Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"056485ab-7f6e-4ae7-84bc-36f35bf53bb1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/056485ab-7f6e-4ae7-84bc-36f35bf53bb1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.320205432Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"056485ab-7f6e-4ae7-84bc-36f35bf53bb1","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"056485ab-7f6e-4ae7-84bc-36f35bf53bb1\" \"--data\" \"/data/regions/056485ab-7f6e-4ae7-84bc-36f35bf53bb1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.32245209Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"056485ab-7f6e-4ae7-84bc-36f35bf53bb1","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.472154166Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions","method":"POST","req_id":"bed69563-01db-4259-b05d-355786597306","remote_addr":"[fd00:1122:3344:101::c]:42098","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":84,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.724986076Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"056485ab-7f6e-4ae7-84bc-36f35bf53bb1","component":"worker"} {"msg":"region 056485ab-7f6e-4ae7-84bc-36f35bf53bb1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.725035797Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.725140328Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.725513191Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.73761062Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.761157963Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.763886305Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/056485ab-7f6e-4ae7-84bc-36f35bf53bb1","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.764436619Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.76449805Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.76453796Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.764574111Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.768409052Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:42.772401044Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:43.017110587Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions","method":"POST","req_id":"7625e4f8-3ea0-445e-a6fe-4de8b999e31a","remote_addr":"[fd00:1122:3344:101::c]:42098","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":64,"response_code":200} {"msg":"region e7f97193-7912-4e82-ae75-82245afe55a4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.402193911Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.402402063Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions","method":"POST","req_id":"c89f35a5-85a6-43b4-a010-a5d537c15c85","remote_addr":"[fd00:1122:3344:101::c]:42098","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":287,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.402427203Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.409039227Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.409055678Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e7f97193-7912-4e82-ae75-82245afe55a4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e7f97193-7912-4e82-ae75-82245afe55a4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.464556642Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"e7f97193-7912-4e82-ae75-82245afe55a4","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e7f97193-7912-4e82-ae75-82245afe55a4\" \"--data\" \"/data/regions/e7f97193-7912-4e82-ae75-82245afe55a4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.464599982Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"e7f97193-7912-4e82-ae75-82245afe55a4","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.538421237Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"e7f97193-7912-4e82-ae75-82245afe55a4","component":"worker"} {"msg":"region e7f97193-7912-4e82-ae75-82245afe55a4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.538458197Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.538550028Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.542348239Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.544847579Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.551956938Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.555117374Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.555338065Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.555382346Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.555421446Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.555457406Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.560461787Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.565429178Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:17:49.595154562Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions","method":"POST","req_id":"e944a7c3-caf4-4a15-98f8-602cc5a11969","remote_addr":"[fd00:1122:3344:101::c]:42098","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":54,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.446759589Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:55431"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.44692542Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions/e7f97193-7912-4e82-ae75-82245afe55a4","method":"GET","req_id":"3d85e057-51bb-4c5c-8567-05b3d3b08d21","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"path is \"/data/regions/e7f97193-7912-4e82-ae75-82245afe55a4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.450282048Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"dataset is oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/e7f97193-7912-4e82-ae75-82245afe55a4","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.462377447Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"region e7f97193-7912-4e82-ae75-82245afe55a4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.470455333Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.470745635Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions/e7f97193-7912-4e82-ae75-82245afe55a4","method":"DELETE","req_id":"ced621c7-2ec3-4ccc-b2be-7425e8494fa0","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20492,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.470768155Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-e7f97193-7912-4e82-ae75-82245afe55a4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.471709003Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.473509088Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions/e7f97193-7912-4e82-ae75-82245afe55a4","method":"GET","req_id":"6c439551-4100-48ae-bcad-f16b65a8e901","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":37,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.483138497Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_c479b44b-6e4c-4a90-ad33-acb415862b6a/crucible/regions/e7f97193-7912-4e82-ae75-82245afe55a4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.490440317Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"region":"e7f97193-7912-4e82-ae75-82245afe55a4","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.536946647Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions/e7f97193-7912-4e82-ae75-82245afe55a4","method":"GET","req_id":"82a51d39-be4c-4733-87bd-d9db7141d534","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":41,"response_code":200} {"msg":"region e7f97193-7912-4e82-ae75-82245afe55a4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.545201055Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-16T02:18:20.705988362Z","hostname":"oxz_crucible_95a5f7b4-19ab-42a6-a947-949d3b885e32","pid":25375,"uri":"/crucible/0/regions/e7f97193-7912-4e82-ae75-82245afe55a4","method":"GET","req_id":"9fa5dbd8-e26c-4733-9f8b-e9368f864048","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":43,"response_code":200}