[ Apr 17 17:30:24 Enabled. ]
[ Apr 17 17:30:24 Rereading configuration. ]
[ Apr 17 17:30:29 Rereading configuration. ]
[ Apr 17 17:30:41 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.466485715Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933}
{"msg":"dataset: \"oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.467166477Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.467179387Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.467184638Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.478927383Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:30:41.751709828Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.158799258Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52150"}
{"msg":"region 2630fac5-9fe7-4850-b392-9b37031a9e39 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.18413445Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.187040325Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.187103178Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions","method":"POST","req_id":"f6e247e5-33cd-4e8c-ab84-452eaa97e0c9","remote_addr":"[fd00:1122:3344:101::a]:52150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14037,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.196277209Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.196294109Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2630fac5-9fe7-4850-b392-9b37031a9e39\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.281517767Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"region":"2630fac5-9fe7-4850-b392-9b37031a9e39","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2630fac5-9fe7-4850-b392-9b37031a9e39\" \"--data\" \"/data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.283798792Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"region":"2630fac5-9fe7-4850-b392-9b37031a9e39","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.551270262Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions","method":"POST","req_id":"b92c8ec2-acf5-438b-a6a5-62fec57fb421","remote_addr":"[fd00:1122:3344:101::a]:52150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.655502974Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"region":"2630fac5-9fe7-4850-b392-9b37031a9e39","component":"worker"}
{"msg":"region 2630fac5-9fe7-4850-b392-9b37031a9e39 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.655574436Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.655755962Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.656071323Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.662764473Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.677458045Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.680737872Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.681436265Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.681498057Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.681537279Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.68157041Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.686572804Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.692396185Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:31:23.830799729Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions","method":"POST","req_id":"3f577cb5-6f46-4799-8f56-3d56f3d62ca4","remote_addr":"[fd00:1122:3344:101::a]:52150","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.582976359Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57237"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.583644141Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","method":"GET","req_id":"6a15a965-525d-4df3-aaaf-ab64fd3c6f7d","remote_addr":"[fd00:1122:3344:101::a]:57237","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"path is \"/data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.587158426Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"dataset is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.59426138Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"snapshot is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39@130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.600780024Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\", \"130e2f81-de6f-4a36-a32d-3397d58895be\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.600812165Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.610220644Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions/2630fac5-9fe7-4850-b392-9b37031a9e39/snapshots/130e2f81-de6f-4a36-a32d-3397d58895be","method":"GET","req_id":"64380ba6-c22c-45f2-8c02-883c76c9da5e","remote_addr":"[fd00:1122:3344:101::a]:57237","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23082,"response_code":200}
{"msg":"path is \"/data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.613546153Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"dataset is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.619592311Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"snapshot is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39@130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.625556407Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\", \"130e2f81-de6f-4a36-a32d-3397d58895be\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.625581178Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2630fac5-9fe7-4850-b392-9b37031a9e39-130e2f81-de6f-4a36-a32d-3397d58895be state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.633715715Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.634314835Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions/2630fac5-9fe7-4850-b392-9b37031a9e39/snapshots/130e2f81-de6f-4a36-a32d-3397d58895be/run","method":"POST","req_id":"4340bb35-4f62-4f38-b023-357b3a008ac9","remote_addr":"[fd00:1122:3344:101::a]:57237","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20777,"response_code":200}
{"msg":"applying SMF actions for region 2630fac5-9fe7-4850-b392-9b37031a9e39 running snapshot 130e2f81-de6f-4a36-a32d-3397d58895be (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.634348246Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2630fac5-9fe7-4850-b392-9b37031a9e39-130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.655135288Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2630fac5-9fe7-4850-b392-9b37031a9e39-130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.661626261Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2630fac5-9fe7-4850-b392-9b37031a9e39-130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.677946137Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39/.zfs/snapshot/130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.678243807Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.678284508Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.678322559Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.67835634Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.678399062Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.683121357Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.688165533Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"worker"}
{"msg":"region 2630fac5-9fe7-4850-b392-9b37031a9e39 running snapshot 130e2f81-de6f-4a36-a32d-3397d58895be state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.688176893Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"path is \"/data/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.882074818Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"dataset is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.888754648Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"datafile"}
{"msg":"snapshot is oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39@130e2f81-de6f-4a36-a32d-3397d58895be","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.894384002Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8ae64d54-16bf-417f-a6e3-5a4eaff73c6a/crucible/regions/2630fac5-9fe7-4850-b392-9b37031a9e39\", \"130e2f81-de6f-4a36-a32d-3397d58895be\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.894414984Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-17T17:41:51.900571166Z","hostname":"oxz_crucible_f04098a6-6652-4e65-bad3-781d2baaa459","pid":29933,"uri":"/crucible/0/regions/2630fac5-9fe7-4850-b392-9b37031a9e39/snapshots/130e2f81-de6f-4a36-a32d-3397d58895be/run","method":"POST","req_id":"b85d1a41-59a4-48dc-b062-c12172eeddca","remote_addr":"[fd00:1122:3344:101::a]:57237","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18563,"response_code":200}