[ May 28 22:15:12 Enabled. ]
[ May 28 22:15:12 Rereading configuration. ]
[ May 28 22:15:16 Rereading configuration. ]
[ May 28 22:15:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.55659956Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614}
{"msg":"dataset: \"oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.557317819Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.557327039Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.557332479Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.569442672Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:15:22.661969318Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.940878912Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:46419"}
{"msg":"region b89916a7-c685-4f17-8953-95f80555899e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.968779283Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.971464755Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.971546865Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"2cb01b35-de84-4e7f-ac35-b97591d69bae","remote_addr":"[fd00:1122:3344:101::c]:46419","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":15790,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.980951637Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:00.980967177Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b89916a7-c685-4f17-8953-95f80555899e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b89916a7-c685-4f17-8953-95f80555899e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.048601738Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"b89916a7-c685-4f17-8953-95f80555899e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b89916a7-c685-4f17-8953-95f80555899e\" \"--data\" \"/data/regions/b89916a7-c685-4f17-8953-95f80555899e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.050894615Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"b89916a7-c685-4f17-8953-95f80555899e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.240176466Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"8717e77c-7abe-4260-8e94-ccca5667cb8d","remote_addr":"[fd00:1122:3344:101::c]:46419","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.431959907Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"b89916a7-c685-4f17-8953-95f80555899e","component":"worker"}
{"msg":"region b89916a7-c685-4f17-8953-95f80555899e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.432032618Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.432236971Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.432623185Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.442632164Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.455885991Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.462792142Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.463931125Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.463981876Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.464024896Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.464059007Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.468892854Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.473448918Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:16:01.502417891Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"cfa5001d-5c9a-4f3f-b8c8-3bbc576327c7","remote_addr":"[fd00:1122:3344:101::c]:46419","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.262072553Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:49423"}
{"msg":"region c31ee296-b817-495d-881f-52cf8571f392 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.262756301Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.263227776Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"a11d7352-f68c-429f-addc-b9119e4e873a","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":1083,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.263247727Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.273642979Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.27367079Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c31ee296-b817-495d-881f-52cf8571f392\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c31ee296-b817-495d-881f-52cf8571f392\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.331811588Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"c31ee296-b817-495d-881f-52cf8571f392","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c31ee296-b817-495d-881f-52cf8571f392\" \"--data\" \"/data/regions/c31ee296-b817-495d-881f-52cf8571f392\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.331870299Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"c31ee296-b817-495d-881f-52cf8571f392","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.440497965Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"c31ee296-b817-495d-881f-52cf8571f392","component":"worker"}
{"msg":"region c31ee296-b817-495d-881f-52cf8571f392 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.440532566Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.440626137Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.460409881Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.464659541Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.474173193Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.476817295Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c31ee296-b817-495d-881f-52cf8571f392","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.477014467Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.477056187Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.477091408Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.477123998Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.480924463Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.485180114Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:16.497802963Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"d5a6c963-d751-4304-976f-93702fa10569","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.958871693Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/b89916a7-c685-4f17-8953-95f80555899e","method":"GET","req_id":"e73c9980-88c2-4551-b8d8-d587a33889f3","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"path is \"/data/regions/b89916a7-c685-4f17-8953-95f80555899e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.96193426Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"dataset is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.968086162Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"snapshot is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e@06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.974216705Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e\", \"06e7891b-38b0-4066-a57a-83a5182d6445\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.974228285Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.983443624Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/b89916a7-c685-4f17-8953-95f80555899e/snapshots/06e7891b-38b0-4066-a57a-83a5182d6445","method":"GET","req_id":"43797ba4-da54-4993-bb05-ae6db2b319bb","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21520,"response_code":200}
{"msg":"path is \"/data/regions/b89916a7-c685-4f17-8953-95f80555899e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.986690863Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"dataset is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.991942265Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"snapshot is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e@06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.997676933Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e\", \"06e7891b-38b0-4066-a57a-83a5182d6445\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:17.997710203Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b89916a7-c685-4f17-8953-95f80555899e-06e7891b-38b0-4066-a57a-83a5182d6445 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.006530198Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"applying SMF actions for region b89916a7-c685-4f17-8953-95f80555899e running snapshot 06e7891b-38b0-4066-a57a-83a5182d6445 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.00668867Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.00670031Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/b89916a7-c685-4f17-8953-95f80555899e/snapshots/06e7891b-38b0-4066-a57a-83a5182d6445/run","method":"POST","req_id":"c5b251f9-60a3-4b9c-8ed1-106b1790e25b","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20012,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-b89916a7-c685-4f17-8953-95f80555899e-06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.0109207Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b89916a7-c685-4f17-8953-95f80555899e-06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.021511945Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b89916a7-c685-4f17-8953-95f80555899e-06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.031672175Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b89916a7-c685-4f17-8953-95f80555899e/.zfs/snapshot/06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.031900958Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.031942839Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.031978819Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.03201569Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.03206235Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.035644122Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.040238097Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"region b89916a7-c685-4f17-8953-95f80555899e running snapshot 06e7891b-38b0-4066-a57a-83a5182d6445 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.040257617Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"path is \"/data/regions/b89916a7-c685-4f17-8953-95f80555899e\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.190579727Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"dataset is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.196584288Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"snapshot is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e@06e7891b-38b0-4066-a57a-83a5182d6445","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.202502928Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/b89916a7-c685-4f17-8953-95f80555899e\", \"06e7891b-38b0-4066-a57a-83a5182d6445\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.202531379Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:18.207916642Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/b89916a7-c685-4f17-8953-95f80555899e/snapshots/06e7891b-38b0-4066-a57a-83a5182d6445/run","method":"POST","req_id":"057d533b-7588-4761-966d-025e726a8b60","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17382,"response_code":200}
{"msg":"region e4fa152d-80fb-4d33-8c21-d0af9f8897b6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.743682229Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.743847561Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"865559c2-fc1b-414f-8d85-91c6de2f4d4e","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":218,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.743875801Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.750225146Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.750246116Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e4fa152d-80fb-4d33-8c21-d0af9f8897b6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.8274296Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"e4fa152d-80fb-4d33-8c21-d0af9f8897b6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e4fa152d-80fb-4d33-8c21-d0af9f8897b6\" \"--data\" \"/data/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.827481161Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"e4fa152d-80fb-4d33-8c21-d0af9f8897b6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.923551758Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"5d2c0074-bfe1-4721-97f9-64fd068c4fad","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.933199163Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"e4fa152d-80fb-4d33-8c21-d0af9f8897b6","component":"worker"}
{"msg":"region e4fa152d-80fb-4d33-8c21-d0af9f8897b6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.933228073Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.933388775Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.935962425Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.939406339Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.947793076Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.951783133Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.952072266Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.952143217Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.952204288Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.952255768Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.956296576Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:22.963242428Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:23.670679385Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions","method":"POST","req_id":"ec244336-1619-4575-a661-fc5883ef78bb","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.793467157Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","method":"GET","req_id":"6d789cdd-88fd-436a-a530-46a16731329d","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":200}
{"msg":"path is \"/data/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.795877535Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"dataset is oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.808316303Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"region e4fa152d-80fb-4d33-8c21-d0af9f8897b6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.815400817Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.816086825Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","method":"DELETE","req_id":"5ed9e268-de1f-4ce8-a46d-190ad6e18a47","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20227,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.816115265Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-e4fa152d-80fb-4d33-8c21-d0af9f8897b6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.817137717Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.81904904Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","method":"GET","req_id":"7e7a61b6-9a12-4e4f-a75c-af1e5a98d11c","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":31,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.828398791Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1ae4cf15-2047-4ba7-bb0a-d60d8fa3219e/crucible/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.836082611Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"region":"e4fa152d-80fb-4d33-8c21-d0af9f8897b6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.878830208Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","method":"GET","req_id":"1a1c350c-865e-4d1e-b687-828a131e6c3f","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"region e4fa152d-80fb-4d33-8c21-d0af9f8897b6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.908547089Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T22:26:47.978229585Z","hostname":"oxz_crucible_930ab65d-f216-42ae-97e9-79ebf92fd923","pid":28614,"uri":"/crucible/0/regions/e4fa152d-80fb-4d33-8c21-d0af9f8897b6","method":"GET","req_id":"65ca629e-a611-4b64-b9ee-eda613783bc5","remote_addr":"[fd00:1122:3344:101::c]:49423","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":41,"response_code":200}