{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.409195915Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50575"} {"msg":"region 804e1022-9f3f-4069-9ec1-834d1661487a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.494518246Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.497713415Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.497727085Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"0c35c31a-93b8-448d-89fd-9637c9d31ea2","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56018,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.526030614Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.526051574Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"804e1022-9f3f-4069-9ec1-834d1661487a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/804e1022-9f3f-4069-9ec1-834d1661487a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.634015978Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"804e1022-9f3f-4069-9ec1-834d1661487a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"804e1022-9f3f-4069-9ec1-834d1661487a\" \"--data\" \"/data/regions/804e1022-9f3f-4069-9ec1-834d1661487a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.636152841Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"804e1022-9f3f-4069-9ec1-834d1661487a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:28.748490131Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"fcc4ac26-70b5-4726-b587-a2b6bec84395","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":52,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.021841749Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"a0b722de-29c6-4339-a961-7f5f2e285753","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.688080432Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"804e1022-9f3f-4069-9ec1-834d1661487a","component":"worker"} {"msg":"region 804e1022-9f3f-4069-9ec1-834d1661487a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.688114352Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.688208084Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.688523698Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.745163896Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.792893668Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.816835617Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/804e1022-9f3f-4069-9ec1-834d1661487a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.817625951Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.817784925Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.817838276Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.817873196Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.841421983Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:29.866091016Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:30.313142792Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"fcde4e24-fd94-43a1-95a5-d8e4471a3a00","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":73,"response_code":200} {"msg":"region 5ef01295-7c3f-4871-b63a-0472eec49c64 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.169487142Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.169645645Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.169662595Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"9712ce74-19a6-476e-8a32-2a4f7fe96be9","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":223,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.176640371Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.176661651Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5ef01295-7c3f-4871-b63a-0472eec49c64\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5ef01295-7c3f-4871-b63a-0472eec49c64\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.254823454Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"5ef01295-7c3f-4871-b63a-0472eec49c64","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5ef01295-7c3f-4871-b63a-0472eec49c64\" \"--data\" \"/data/regions/5ef01295-7c3f-4871-b63a-0472eec49c64\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.254857855Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"5ef01295-7c3f-4871-b63a-0472eec49c64","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.486555313Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"5ef01295-7c3f-4871-b63a-0472eec49c64","component":"worker"} {"msg":"region 5ef01295-7c3f-4871-b63a-0472eec49c64 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.486608333Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.486684224Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.487009079Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.49957305Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.520137261Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.532536309Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.532809323Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.532860034Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.532900734Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.532939395Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.535409852Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions","method":"POST","req_id":"5c4b026e-08d2-452b-b885-9a13e4b4eca4","remote_addr":"[fd00:1122:3344:101::a]:50575","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":51,"response_code":200} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.545359103Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:46:38.57953251Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.431311919Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44913"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.431439371Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","method":"GET","req_id":"10efa3d5-1210-496e-9e5b-4b848feaa232","remote_addr":"[fd00:1122:3344:101::a]:44913","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":200} {"msg":"path is \"/data/regions/5ef01295-7c3f-4871-b63a-0472eec49c64\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.433913838Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"dataset is oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.445393012Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"region 5ef01295-7c3f-4871-b63a-0472eec49c64 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.453539235Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.453818669Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.45383677Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","method":"DELETE","req_id":"5747cb54-9ee1-42ac-8ec1-d5bb409a5ae8","remote_addr":"[fd00:1122:3344:101::a]:44913","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19927,"response_code":204} {"msg":"disabling downstairs instance: downstairs-5ef01295-7c3f-4871-b63a-0472eec49c64 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.454431429Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.456406558Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","method":"GET","req_id":"9d7004f0-f042-4412-a2c2-de1c223d5dc4","remote_addr":"[fd00:1122:3344:101::a]:44913","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.478520603Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_3dbdbd7d-9613-4ef5-8c5e-977b0a040901/crucible/regions/5ef01295-7c3f-4871-b63a-0472eec49c64\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.486704557Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"region":"5ef01295-7c3f-4871-b63a-0472eec49c64","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.513493433Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","method":"GET","req_id":"264fa9a4-cc97-4017-924a-ec23c74f7aad","remote_addr":"[fd00:1122:3344:101::a]:44913","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":57,"response_code":200} {"msg":"region 5ef01295-7c3f-4871-b63a-0472eec49c64 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.573248667Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:47:11.621781622Z","hostname":"oxz_crucible_b34b1d3d-db45-49ab-80d5-a5881c5e1cad","pid":25553,"uri":"/crucible/0/regions/5ef01295-7c3f-4871-b63a-0472eec49c64","method":"GET","req_id":"927c0168-b0a1-4e21-8810-57735145c1be","remote_addr":"[fd00:1122:3344:101::a]:44913","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":51,"response_code":200}