[ May 28 06:15:21 Enabled. ]
[ May 28 06:15:21 Rereading configuration. ]
[ May 28 06:15:23 Rereading configuration. ]
[ May 28 06:15:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.769712075Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958}
{"msg":"dataset: \"oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.770532364Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.770548815Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.770554965Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.789750192Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:15:26.86075152Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.662089325Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:49029"}
{"msg":"region b1225eef-483d-4561-a72d-9cd008664d65 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.710496366Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.714986043Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.715035254Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions","method":"POST","req_id":"b7f69930-57fd-41d3-a26f-d89d03a625ab","remote_addr":"[fd00:1122:3344:101::c]:49029","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19134,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.721716843Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.721746934Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b1225eef-483d-4561-a72d-9cd008664d65\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b1225eef-483d-4561-a72d-9cd008664d65\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.767502972Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"region":"b1225eef-483d-4561-a72d-9cd008664d65","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b1225eef-483d-4561-a72d-9cd008664d65\" \"--data\" \"/data/regions/b1225eef-483d-4561-a72d-9cd008664d65\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.770091254Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"region":"b1225eef-483d-4561-a72d-9cd008664d65","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.900820384Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"region":"b1225eef-483d-4561-a72d-9cd008664d65","component":"worker"}
{"msg":"region b1225eef-483d-4561-a72d-9cd008664d65 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.900867905Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.900972507Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.901292515Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.908197169Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.918926064Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.920334918Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.921040445Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.921103806Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.921153957Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.921190818Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.923524714Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.925934041Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:16:46.939956664Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions","method":"POST","req_id":"4ee8c04d-39f7-4efb-8716-2f7f322334ec","remote_addr":"[fd00:1122:3344:101::c]:49029","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.520924645Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:41404"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.521045317Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions/b1225eef-483d-4561-a72d-9cd008664d65","method":"GET","req_id":"fd477325-21b4-42ce-8446-384f876250b9","remote_addr":"[fd00:1122:3344:101::c]:41404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"path is \"/data/regions/b1225eef-483d-4561-a72d-9cd008664d65\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.525296719Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"dataset is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.531604179Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"snapshot is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65@1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.537805916Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65\", \"1889ab9d-5d4d-471a-8f84-4ec5abe39789\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.537830327Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.546608456Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions/b1225eef-483d-4561-a72d-9cd008664d65/snapshots/1889ab9d-5d4d-471a-8f84-4ec5abe39789","method":"GET","req_id":"7ae0aa89-cf16-4342-9719-fec9b32240f2","remote_addr":"[fd00:1122:3344:101::c]:41404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21330,"response_code":200}
{"msg":"path is \"/data/regions/b1225eef-483d-4561-a72d-9cd008664d65\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.550372535Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"dataset is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.555908077Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"snapshot is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65@1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.561046159Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65\", \"1889ab9d-5d4d-471a-8f84-4ec5abe39789\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.56106298Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b1225eef-483d-4561-a72d-9cd008664d65-1889ab9d-5d4d-471a-8f84-4ec5abe39789 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.567234526Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.567979794Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions/b1225eef-483d-4561-a72d-9cd008664d65/snapshots/1889ab9d-5d4d-471a-8f84-4ec5abe39789/run","method":"POST","req_id":"d3ede608-0718-4f8f-ab1a-204fc2ca21af","remote_addr":"[fd00:1122:3344:101::c]:41404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17629,"response_code":200}
{"msg":"applying SMF actions for region b1225eef-483d-4561-a72d-9cd008664d65 running snapshot 1889ab9d-5d4d-471a-8f84-4ec5abe39789 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.568016915Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-b1225eef-483d-4561-a72d-9cd008664d65-1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.581229819Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b1225eef-483d-4561-a72d-9cd008664d65-1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.589070036Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b1225eef-483d-4561-a72d-9cd008664d65-1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.596990834Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b1225eef-483d-4561-a72d-9cd008664d65/.zfs/snapshot/1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.597285471Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.597319452Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.597364683Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.597400594Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.597442855Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.599661807Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.602588317Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"worker"}
{"msg":"region b1225eef-483d-4561-a72d-9cd008664d65 running snapshot 1889ab9d-5d4d-471a-8f84-4ec5abe39789 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.602611698Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"path is \"/data/regions/b1225eef-483d-4561-a72d-9cd008664d65\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.872971739Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"dataset is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.879414312Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"datafile"}
{"msg":"snapshot is oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65@1889ab9d-5d4d-471a-8f84-4ec5abe39789","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.886529881Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8cdcdcc0-2868-4016-9b0a-36837a5b3e6f/crucible/regions/b1225eef-483d-4561-a72d-9cd008664d65\", \"1889ab9d-5d4d-471a-8f84-4ec5abe39789\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.886563862Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-28T06:27:44.891796376Z","hostname":"oxz_crucible_7b96b3e0-66b7-4bf9-be03-bd6a09c3925a","pid":26958,"uri":"/crucible/0/regions/b1225eef-483d-4561-a72d-9cd008664d65/snapshots/1889ab9d-5d4d-471a-8f84-4ec5abe39789/run","method":"POST","req_id":"60394d53-6015-4a6a-9c6b-0263ea97f60a","remote_addr":"[fd00:1122:3344:101::c]:41404","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18868,"response_code":200}