{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.456002561Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:52336"} {"msg":"region 116d6108-134d-4fd2-8a38-cd3a221a4579 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.456073932Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.456819854Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions","method":"POST","req_id":"8d701740-8622-4373-ae58-f9097d97a95c","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":777,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.456841034Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.46680707Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.46682507Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"116d6108-134d-4fd2-8a38-cd3a221a4579\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/116d6108-134d-4fd2-8a38-cd3a221a4579\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.524398131Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"region":"116d6108-134d-4fd2-8a38-cd3a221a4579","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"116d6108-134d-4fd2-8a38-cd3a221a4579\" \"--data\" \"/data/regions/116d6108-134d-4fd2-8a38-cd3a221a4579\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.524433812Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"region":"116d6108-134d-4fd2-8a38-cd3a221a4579","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.632177658Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"region":"116d6108-134d-4fd2-8a38-cd3a221a4579","component":"worker"} {"msg":"region 116d6108-134d-4fd2-8a38-cd3a221a4579 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.632210379Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.632320151Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.654695421Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.663040452Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.67314758Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.675921373Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/116d6108-134d-4fd2-8a38-cd3a221a4579","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.676224208Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.676280489Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.676324799Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.67636994Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.679663062Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.684118761Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:04.801693642Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions","method":"POST","req_id":"b22c7368-1fbf-4251-a7ec-da6172a6ae24","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.927143487Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions/0102f447-9442-40f1-94c0-eae012f5fbc1","method":"GET","req_id":"2c72b633-f097-4f2e-af5a-359a83f10979","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":74,"response_code":200} {"msg":"path is \"/data/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.930075853Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"dataset is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.936107027Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"snapshot is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1@75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.942896953Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\", \"75bfd22a-0749-4f53-ac62-7211f328ff06\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.942914934Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.951088622Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions/0102f447-9442-40f1-94c0-eae012f5fbc1/snapshots/75bfd22a-0749-4f53-ac62-7211f328ff06","method":"GET","req_id":"97e5195d-5e4c-43f4-93bf-4f33b18b02d7","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":21038,"response_code":200} {"msg":"path is \"/data/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.954572696Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"dataset is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.961686928Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"snapshot is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1@75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.967636001Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\", \"75bfd22a-0749-4f53-ac62-7211f328ff06\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.967651421Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 0102f447-9442-40f1-94c0-eae012f5fbc1-75bfd22a-0749-4f53-ac62-7211f328ff06 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.974990236Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.975124848Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions/0102f447-9442-40f1-94c0-eae012f5fbc1/snapshots/75bfd22a-0749-4f53-ac62-7211f328ff06/run","method":"POST","req_id":"1bbfd91f-e865-4dbb-84fc-0e414237dfc2","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20573,"response_code":200} {"msg":"applying SMF actions for region 0102f447-9442-40f1-94c0-eae012f5fbc1 running snapshot 75bfd22a-0749-4f53-ac62-7211f328ff06 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.975141948Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-0102f447-9442-40f1-94c0-eae012f5fbc1-75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.979596788Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0102f447-9442-40f1-94c0-eae012f5fbc1-75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:05.992360488Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0102f447-9442-40f1-94c0-eae012f5fbc1-75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.00148956Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0102f447-9442-40f1-94c0-eae012f5fbc1/.zfs/snapshot/75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.001774194Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.001814505Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.001853006Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.001885296Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.001927137Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.005800877Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.010074634Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"worker"} {"msg":"region 0102f447-9442-40f1-94c0-eae012f5fbc1 running snapshot 75bfd22a-0749-4f53-ac62-7211f328ff06 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.010087535Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"path is \"/data/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.290207049Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"dataset is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.29667769Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"datafile"} {"msg":"snapshot is oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1@75bfd22a-0749-4f53-ac62-7211f328ff06","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.302826946Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_38af5472-283e-4d15-806a-ab59eaa7d71e/crucible/regions/0102f447-9442-40f1-94c0-eae012f5fbc1\", \"75bfd22a-0749-4f53-ac62-7211f328ff06\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.302859737Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-02T22:37:06.31010238Z","hostname":"oxz_crucible_78646757-89ad-4d17-9fdd-1b93cf1b3883","pid":23343,"uri":"/crucible/0/regions/0102f447-9442-40f1-94c0-eae012f5fbc1/snapshots/75bfd22a-0749-4f53-ac62-7211f328ff06/run","method":"POST","req_id":"323f575f-0428-443d-9eb2-5d5592e8b496","remote_addr":"[fd00:1122:3344:101::c]:52336","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19934,"response_code":200}