{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.698899005Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64614"} {"msg":"region c3cba4a6-743d-494f-8926-328f0413ee9d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.698999527Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.69965404Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions","method":"POST","req_id":"654e99ac-8084-4548-a611-fc546b42db45","remote_addr":"[fd00:1122:3344:101::a]:64614","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":686,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.69966938Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.708264102Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.708281743Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c3cba4a6-743d-494f-8926-328f0413ee9d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c3cba4a6-743d-494f-8926-328f0413ee9d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.76690505Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c3cba4a6-743d-494f-8926-328f0413ee9d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c3cba4a6-743d-494f-8926-328f0413ee9d\" \"--data\" \"/data/regions/c3cba4a6-743d-494f-8926-328f0413ee9d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:51.766939761Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c3cba4a6-743d-494f-8926-328f0413ee9d","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.074260116Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c3cba4a6-743d-494f-8926-328f0413ee9d","component":"worker"} {"msg":"region c3cba4a6-743d-494f-8926-328f0413ee9d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.074317457Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.074415759Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.076212073Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.080051165Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions","method":"POST","req_id":"0875225f-c6ae-48eb-9b39-95a66d9a3fbf","remote_addr":"[fd00:1122:3344:101::a]:64614","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":75,"response_code":200} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.0871896Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.112859435Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.116967443Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c3cba4a6-743d-494f-8926-328f0413ee9d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.11737451Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.117424691Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.117481062Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.117532453Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.121888546Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:52.126426301Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"region c92e68c8-14f9-4acb-a762-2b55573148a3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.330110522Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.330293745Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions","method":"POST","req_id":"b380cd14-4ab6-48a3-92d0-ccf49dcb3afc","remote_addr":"[fd00:1122:3344:101::a]:64614","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":231,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.330311266Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.337162745Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.337183575Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c92e68c8-14f9-4acb-a762-2b55573148a3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c92e68c8-14f9-4acb-a762-2b55573148a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.39670009Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c92e68c8-14f9-4acb-a762-2b55573148a3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c92e68c8-14f9-4acb-a762-2b55573148a3\" \"--data\" \"/data/regions/c92e68c8-14f9-4acb-a762-2b55573148a3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.39673294Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c92e68c8-14f9-4acb-a762-2b55573148a3","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.481613064Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c92e68c8-14f9-4acb-a762-2b55573148a3","component":"worker"} {"msg":"region c92e68c8-14f9-4acb-a762-2b55573148a3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.481649504Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.481757526Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.483923907Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions","method":"POST","req_id":"3f903c54-abd3-4b70-9605-2692b173b0b3","remote_addr":"[fd00:1122:3344:101::a]:64614","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"creating missing downstairs instance downstairs-c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.486353113Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.495051598Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.502149282Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.505865962Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.506066016Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.506107057Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.506142287Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.506185888Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.510167083Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:16:58.514918383Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.369193661Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47350"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.369308704Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","method":"GET","req_id":"c335eb84-fc50-4ada-92be-97471588398e","remote_addr":"[fd00:1122:3344:101::a]:47350","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":39,"response_code":200} {"msg":"path is \"/data/regions/c92e68c8-14f9-4acb-a762-2b55573148a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.373142946Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"dataset is oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.384333737Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"region c92e68c8-14f9-4acb-a762-2b55573148a3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.392540572Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.392690555Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","method":"DELETE","req_id":"fdb5aa4c-c729-4513-9134-09189df6ff87","remote_addr":"[fd00:1122:3344:101::a]:47350","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19577,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.392725056Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-c92e68c8-14f9-4acb-a762-2b55573148a3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.394471119Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.395784834Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","method":"GET","req_id":"a155275d-f53f-428b-8d5f-08baee5b1f39","remote_addr":"[fd00:1122:3344:101::a]:47350","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.419070744Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_7a2555d0-43a0-40ba-b56a-eacc9b56531c/crucible/regions/c92e68c8-14f9-4acb-a762-2b55573148a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.426644667Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"region":"c92e68c8-14f9-4acb-a762-2b55573148a3","component":"worker"} {"msg":"region c92e68c8-14f9-4acb-a762-2b55573148a3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.511163623Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-03T06:17:32.540076089Z","hostname":"oxz_crucible_44afb661-d860-4334-9d6d-b08ff7c0869e","pid":22459,"uri":"/crucible/0/regions/c92e68c8-14f9-4acb-a762-2b55573148a3","method":"GET","req_id":"6f0cb820-1c0e-47f6-accf-e335b1742f53","remote_addr":"[fd00:1122:3344:101::a]:47350","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":50,"response_code":200}