{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.901335239Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64697"} {"msg":"region 73361ff0-ed1e-4fed-b05b-4eddee4846d6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.901525213Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.902086883Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.902096013Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions","method":"POST","req_id":"b18f61b8-92b9-4154-90b5-4ac4bc77f829","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":603,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.910540174Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.910559114Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"73361ff0-ed1e-4fed-b05b-4eddee4846d6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/73361ff0-ed1e-4fed-b05b-4eddee4846d6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.994270343Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"region":"73361ff0-ed1e-4fed-b05b-4eddee4846d6","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"73361ff0-ed1e-4fed-b05b-4eddee4846d6\" \"--data\" \"/data/regions/73361ff0-ed1e-4fed-b05b-4eddee4846d6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:21.994308014Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"region":"73361ff0-ed1e-4fed-b05b-4eddee4846d6","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.151006469Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions","method":"POST","req_id":"b69fe2e7-9aef-4ffd-974b-95958fab2522","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.211581682Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"region":"73361ff0-ed1e-4fed-b05b-4eddee4846d6","component":"worker"} {"msg":"region 73361ff0-ed1e-4fed-b05b-4eddee4846d6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.211606293Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.211724045Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.242291602Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.261329513Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.29688425Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.305083856Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/73361ff0-ed1e-4fed-b05b-4eddee4846d6","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.305388102Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.305431692Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.305467833Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.305502344Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.321290536Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.335717445Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:22.738243431Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions","method":"POST","req_id":"3fe883e5-4e60-4cc1-8dbb-e308656e39c3","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.529013119Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f","method":"GET","req_id":"2883083d-3b92-477f-95fd-b15292afd3de","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":78,"response_code":200} {"msg":"path is \"/data/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.532164665Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"dataset is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.53856207Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"snapshot is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f@2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.54564727Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\", \"2664ae10-3efe-4ec6-a8df-b585d3cc4e25\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.545666705Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.565429451Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f/snapshots/2664ae10-3efe-4ec6-a8df-b585d3cc4e25","method":"GET","req_id":"ed5e4ee9-e63c-48ca-b511-b3a2c087bbe9","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":33286,"response_code":200} {"msg":"path is \"/data/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.568568967Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"dataset is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.575131224Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"snapshot is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f@2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.581738363Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\", \"2664ae10-3efe-4ec6-a8df-b585d3cc4e25\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.581767723Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 5e68b7a0-2691-44a1-be96-977ca6be0d8f-2664ae10-3efe-4ec6-a8df-b585d3cc4e25 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.58940804Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"applying SMF actions for region 5e68b7a0-2691-44a1-be96-977ca6be0d8f running snapshot 2664ae10-3efe-4ec6-a8df-b585d3cc4e25 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.589532582Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.589548833Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f/snapshots/2664ae10-3efe-4ec6-a8df-b585d3cc4e25/run","method":"POST","req_id":"df4a6809-72ce-4076-af1b-32ff3dbcc24f","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20988,"response_code":200} {"msg":"creating missing snapshot instance snapshot-5e68b7a0-2691-44a1-be96-977ca6be0d8f-2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.593718427Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5e68b7a0-2691-44a1-be96-977ca6be0d8f-2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.612040215Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5e68b7a0-2691-44a1-be96-977ca6be0d8f-2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.642673144Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f/.zfs/snapshot/2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.642970639Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.64301638Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.64305317Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.643088191Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.643141232Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.661198995Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.674753168Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"worker"} {"msg":"region 5e68b7a0-2691-44a1-be96-977ca6be0d8f running snapshot 2664ae10-3efe-4ec6-a8df-b585d3cc4e25 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.674764808Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"path is \"/data/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.722827559Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"dataset is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.729016769Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"datafile"} {"msg":"snapshot is oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f@2664ae10-3efe-4ec6-a8df-b585d3cc4e25","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.734346515Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_fd03613b-dbb6-4e55-9d89-5457c9dc3253/crucible/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f\", \"2664ae10-3efe-4ec6-a8df-b585d3cc4e25\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.734365315Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-15T01:34:24.739884114Z","hostname":"oxz_crucible_83f2d1ec-81e9-4d25-9739-105ca73a918f","pid":24000,"uri":"/crucible/0/regions/5e68b7a0-2691-44a1-be96-977ca6be0d8f/snapshots/2664ae10-3efe-4ec6-a8df-b585d3cc4e25/run","method":"POST","req_id":"b9612ce5-1282-4208-a7aa-f6aaa474e973","remote_addr":"[fd00:1122:3344:101::a]:64697","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17084,"response_code":200}