{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.863260729Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62547"} {"msg":"region e436b6f1-3376-4067-95ab-5979a4fd06ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.863390931Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.863964939Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions","method":"POST","req_id":"cc9929d8-e637-4543-8de1-90fc11b86fb9","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":621,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.863977029Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.877557397Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.877579038Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e436b6f1-3376-4067-95ab-5979a4fd06ed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e436b6f1-3376-4067-95ab-5979a4fd06ed\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.973857842Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"region":"e436b6f1-3376-4067-95ab-5979a4fd06ed","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e436b6f1-3376-4067-95ab-5979a4fd06ed\" \"--data\" \"/data/regions/e436b6f1-3376-4067-95ab-5979a4fd06ed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:25.973891693Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"region":"e436b6f1-3376-4067-95ab-5979a4fd06ed","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.203952136Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"region":"e436b6f1-3376-4067-95ab-5979a4fd06ed","component":"worker"} {"msg":"region e436b6f1-3376-4067-95ab-5979a4fd06ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.203985347Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.204100849Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.215164278Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions","method":"POST","req_id":"58eb82ed-b184-4188-bea1-90e80aafce80","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":200} {"msg":"creating missing downstairs instance downstairs-e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.223459755Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.244124782Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.275542293Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.282451118Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e436b6f1-3376-4067-95ab-5979a4fd06ed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.282735953Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.282780303Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.282821384Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.282857625Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.292823307Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:26.299961857Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.381099655Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69","method":"GET","req_id":"bba83f3f-48dc-403b-9dfa-00340109ad05","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":200} {"msg":"path is \"/data/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.384835532Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"dataset is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.391504814Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"snapshot is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69@943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.397576707Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\", \"943e7588-f544-420d-994f-04a46104d1ac\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.397596858Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.406075987Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69/snapshots/943e7588-f544-420d-994f-04a46104d1ac","method":"GET","req_id":"f28d8fff-4878-4100-8241-46c06df217b5","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21257,"response_code":200} {"msg":"path is \"/data/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.409817975Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"dataset is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.415909068Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"snapshot is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69@943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.421322611Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\", \"943e7588-f544-420d-994f-04a46104d1ac\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.421344211Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 1b8d9d99-c44c-4049-a258-955c2f00dc69-943e7588-f544-420d-994f-04a46104d1ac state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.430149496Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.430272378Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69/snapshots/943e7588-f544-420d-994f-04a46104d1ac/run","method":"POST","req_id":"4e699dda-0b72-4387-b7f0-0567c9eb6781","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20479,"response_code":200} {"msg":"applying SMF actions for region 1b8d9d99-c44c-4049-a258-955c2f00dc69 running snapshot 943e7588-f544-420d-994f-04a46104d1ac (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.430290809Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-1b8d9d99-c44c-4049-a258-955c2f00dc69-943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.435040526Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1b8d9d99-c44c-4049-a258-955c2f00dc69-943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.452474238Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1b8d9d99-c44c-4049-a258-955c2f00dc69-943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470118608Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69/.zfs/snapshot/943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470334162Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470376642Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470413253Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470446784Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.470489024Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.475253447Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.482506178Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"worker"} {"msg":"region 1b8d9d99-c44c-4049-a258-955c2f00dc69 running snapshot 943e7588-f544-420d-994f-04a46104d1ac state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.482515908Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"path is \"/data/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.668835682Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"dataset is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.674809683Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"datafile"} {"msg":"snapshot is oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69@943e7588-f544-420d-994f-04a46104d1ac","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.681425055Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1fb74e6c-7654-4598-9eff-3750bf54efed/crucible/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69\", \"943e7588-f544-420d-994f-04a46104d1ac\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.681445155Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-05T18:02:27.68700933Z","hostname":"oxz_crucible_4d82448c-708a-4b6b-a367-d813c8acaca0","pid":22429,"uri":"/crucible/0/regions/1b8d9d99-c44c-4049-a258-955c2f00dc69/snapshots/943e7588-f544-420d-994f-04a46104d1ac/run","method":"POST","req_id":"f924456b-25cb-43a5-b4da-106de493c170","remote_addr":"[fd00:1122:3344:101::a]:62547","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18168,"response_code":200}