{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.065419843Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64872"} {"msg":"region f7a9f095-fd5e-429b-9a95-994401278a47 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.065580676Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.066211665Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.066227025Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions","method":"POST","req_id":"f71d31a0-bb50-4f2b-8422-561790ffa097","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":682,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.075380771Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.075391601Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f7a9f095-fd5e-429b-9a95-994401278a47\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f7a9f095-fd5e-429b-9a95-994401278a47\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.121922373Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"region":"f7a9f095-fd5e-429b-9a95-994401278a47","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f7a9f095-fd5e-429b-9a95-994401278a47\" \"--data\" \"/data/regions/f7a9f095-fd5e-429b-9a95-994401278a47\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.121957764Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"region":"f7a9f095-fd5e-429b-9a95-994401278a47","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.198902127Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"region":"f7a9f095-fd5e-429b-9a95-994401278a47","component":"worker"} {"msg":"region f7a9f095-fd5e-429b-9a95-994401278a47 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.198936068Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.19905435Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.218372987Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.228794712Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.240653098Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.243992458Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f7a9f095-fd5e-429b-9a95-994401278a47","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.244286802Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.244332452Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.244372863Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.244409174Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.248517665Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.254432283Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.432381368Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions","method":"POST","req_id":"b9dc5486-667b-4a5f-8239-fd6a4bdfda75","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":57,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.816658959Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions/baa038f6-9523-4861-8583-02ea0dfa3f94","method":"GET","req_id":"ebf24d55-3f5b-44f2-96fa-d41ad80955cb","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":65,"response_code":200} {"msg":"path is \"/data/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.821467671Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"dataset is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.827662943Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"snapshot is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94@53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.834307542Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\", \"53a934f6-d542-4134-bed9-a4f304b74bc8\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.834318792Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.843771342Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions/baa038f6-9523-4861-8583-02ea0dfa3f94/snapshots/53a934f6-d542-4134-bed9-a4f304b74bc8","method":"GET","req_id":"68be7dee-7bdc-4737-9c30-cdebf78663a7","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22328,"response_code":200} {"msg":"path is \"/data/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.847363556Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"dataset is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.853804962Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"snapshot is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94@53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.859679789Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\", \"53a934f6-d542-4134-bed9-a4f304b74bc8\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.859699659Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot baa038f6-9523-4861-8583-02ea0dfa3f94-53a934f6-d542-4134-bed9-a4f304b74bc8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.86783882Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"applying SMF actions for region baa038f6-9523-4861-8583-02ea0dfa3f94 running snapshot 53a934f6-d542-4134-bed9-a4f304b74bc8 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.867976212Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.867986293Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions/baa038f6-9523-4861-8583-02ea0dfa3f94/snapshots/53a934f6-d542-4134-bed9-a4f304b74bc8/run","method":"POST","req_id":"1ea07c8b-7aec-45dc-80d1-a62d34a1bfbe","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20627,"response_code":200} {"msg":"creating missing snapshot instance snapshot-baa038f6-9523-4861-8583-02ea0dfa3f94-53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.872777644Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-baa038f6-9523-4861-8583-02ea0dfa3f94-53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.88127775Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-baa038f6-9523-4861-8583-02ea0dfa3f94-53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891485862Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/baa038f6-9523-4861-8583-02ea0dfa3f94/.zfs/snapshot/53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891784196Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891826837Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891867957Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891902538Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.891948989Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.895798416Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.900424855Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"worker"} {"msg":"region baa038f6-9523-4861-8583-02ea0dfa3f94 running snapshot 53a934f6-d542-4134-bed9-a4f304b74bc8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:30.900434045Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"path is \"/data/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:31.012646912Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"dataset is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:31.018828914Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"datafile"} {"msg":"snapshot is oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94@53a934f6-d542-4134-bed9-a4f304b74bc8","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:31.024757272Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_946c6df9-1ffe-47da-95af-db26fa21cdcf/crucible/regions/baa038f6-9523-4861-8583-02ea0dfa3f94\", \"53a934f6-d542-4134-bed9-a4f304b74bc8\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:31.024775452Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T02:34:31.030860473Z","hostname":"oxz_crucible_524bb265-0607-46e0-ac7e-deab269304d7","pid":23952,"uri":"/crucible/0/regions/baa038f6-9523-4861-8583-02ea0dfa3f94/snapshots/53a934f6-d542-4134-bed9-a4f304b74bc8/run","method":"POST","req_id":"64e799ef-5485-44f9-b5b7-f02291ac964b","remote_addr":"[fd00:1122:3344:101::a]:64872","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18244,"response_code":200}