{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.521822316Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:55220"} {"msg":"region 85f79e90-c326-4536-947c-9f198116383d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.569024565Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.57251727Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.572533461Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions","method":"POST","req_id":"066286fa-95db-4d21-9fe0-ca580358988d","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20354,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.581200904Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.581221255Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"85f79e90-c326-4536-947c-9f198116383d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/85f79e90-c326-4536-947c-9f198116383d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.639159994Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"85f79e90-c326-4536-947c-9f198116383d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"85f79e90-c326-4536-947c-9f198116383d\" \"--data\" \"/data/regions/85f79e90-c326-4536-947c-9f198116383d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.641413933Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"85f79e90-c326-4536-947c-9f198116383d","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:57.886057759Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions","method":"POST","req_id":"a2e95dfb-61ea-4074-8772-439120ef5c98","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.059036612Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"85f79e90-c326-4536-947c-9f198116383d","component":"worker"} {"msg":"region 85f79e90-c326-4536-947c-9f198116383d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.059091419Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.059230743Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.059781666Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.0799259Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.118503225Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.125345975Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/85f79e90-c326-4536-947c-9f198116383d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.126372839Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.12645881Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.12651821Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.126569821Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.134964701Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.144504086Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:40:58.614522505Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions","method":"POST","req_id":"1e5ff0b6-30b4-4564-a6cb-1edd7f286feb","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region bfd829aa-09f4-40eb-b74a-47974c327522 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.246605657Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.246773229Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.246787639Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions","method":"POST","req_id":"27705b31-6096-4919-861a-068ce21b3263","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":222,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.253935703Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.253957353Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bfd829aa-09f4-40eb-b74a-47974c327522\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bfd829aa-09f4-40eb-b74a-47974c327522\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.305151864Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"bfd829aa-09f4-40eb-b74a-47974c327522","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bfd829aa-09f4-40eb-b74a-47974c327522\" \"--data\" \"/data/regions/bfd829aa-09f4-40eb-b74a-47974c327522\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.305183355Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"bfd829aa-09f4-40eb-b74a-47974c327522","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.379982905Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"bfd829aa-09f4-40eb-b74a-47974c327522","component":"worker"} {"msg":"region bfd829aa-09f4-40eb-b74a-47974c327522 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.380030245Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.380137947Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.384357212Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.387437233Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.394170671Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.397631906Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.397843219Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.39789727Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.39794216Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.397985071Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.401969983Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.406946588Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:05.602357708Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions","method":"POST","req_id":"a294d719-7531-44ca-a196-a74972c0c198","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.44184596Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions/bfd829aa-09f4-40eb-b74a-47974c327522","method":"GET","req_id":"da10e729-13b1-42a2-be14-70e8655a51d5","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":200} {"msg":"path is \"/data/regions/bfd829aa-09f4-40eb-b74a-47974c327522\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.445912103Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"dataset is oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/bfd829aa-09f4-40eb-b74a-47974c327522","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.456246058Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"region bfd829aa-09f4-40eb-b74a-47974c327522 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.463130529Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.463272201Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.463291141Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions/bfd829aa-09f4-40eb-b74a-47974c327522","method":"DELETE","req_id":"ab9ab911-788d-499f-a5d9-5da8256bbe64","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17393,"response_code":204} {"msg":"disabling downstairs instance: downstairs-bfd829aa-09f4-40eb-b74a-47974c327522 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.464337764Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.466438092Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions/bfd829aa-09f4-40eb-b74a-47974c327522","method":"GET","req_id":"b8c548af-6f12-4df1-b897-175d908e73ee","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.477174313Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_561bc310-6f83-4e6d-82fd-ecacc460531b/crucible/regions/bfd829aa-09f4-40eb-b74a-47974c327522\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.485644514Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"region":"bfd829aa-09f4-40eb-b74a-47974c327522","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.545094082Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions/bfd829aa-09f4-40eb-b74a-47974c327522","method":"GET","req_id":"576740aa-80b6-4a1e-8418-6d41db70c8e1","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":200} {"msg":"region bfd829aa-09f4-40eb-b74a-47974c327522 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.546066875Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-24T18:41:30.665416409Z","hostname":"oxz_crucible_2da551dd-6760-4778-8a0c-d40a4bb7036e","pid":23848,"uri":"/crucible/0/regions/bfd829aa-09f4-40eb-b74a-47974c327522","method":"GET","req_id":"0451ff7f-a3c1-42a5-8b7b-fe5e2deb7265","remote_addr":"[fd00:1122:3344:101::b]:55220","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":48,"response_code":200}