[ Aug 29 00:00:34 Enabled. ]
[ Aug 29 00:00:34 Rereading configuration. ]
[ Aug 29 00:00:36 Rereading configuration. ]
[ Aug 29 00:00:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.668375387Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965}
{"msg":"dataset: \"oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.668586162Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.668600532Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.668608082Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.691915664Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:00:40.793880057Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.776936904Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:50711"}
{"msg":"region 536352b3-156c-4018-875f-b07c78fab143 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.819858665Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.824964167Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.830076108Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions","method":"POST","req_id":"86121711-b205-4a59-b895-560f43719180","remote_addr":"[fd00:1122:3344:101::b]:50711","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19896,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.835846962Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.835871693Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"536352b3-156c-4018-875f-b07c78fab143\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/536352b3-156c-4018-875f-b07c78fab143\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.879731873Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"region":"536352b3-156c-4018-875f-b07c78fab143","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"536352b3-156c-4018-875f-b07c78fab143\" \"--data\" \"/data/regions/536352b3-156c-4018-875f-b07c78fab143\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:50.882057179Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"region":"536352b3-156c-4018-875f-b07c78fab143","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.005216212Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions","method":"POST","req_id":"1e944c20-93bf-4dfd-8430-a4ddae96f420","remote_addr":"[fd00:1122:3344:101::b]:50711","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.21985254Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"region":"536352b3-156c-4018-875f-b07c78fab143","component":"worker"}
{"msg":"region 536352b3-156c-4018-875f-b07c78fab143 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.224831189Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.225120214Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.228393599Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.235544101Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.250553759Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.252157221Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.252976547Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.253035318Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.253089119Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.25312932Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.255848094Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.259276642Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:01:51.709937972Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions","method":"POST","req_id":"5af346d1-7995-4b38-80bf-670f03509228","remote_addr":"[fd00:1122:3344:101::b]:50711","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.851696398Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38821"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.851818871Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions/536352b3-156c-4018-875f-b07c78fab143","method":"GET","req_id":"0edbb8cd-d96f-4360-8783-193d84e2ef23","remote_addr":"[fd00:1122:3344:101::b]:38821","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/536352b3-156c-4018-875f-b07c78fab143\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.855194388Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"dataset is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.86135148Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"snapshot is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143@88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.866068854Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143\", \"88d153fb-5064-4ed9-94f3-4fdec70b8aaa\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.866086144Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.878746205Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions/536352b3-156c-4018-875f-b07c78fab143/snapshots/88d153fb-5064-4ed9-94f3-4fdec70b8aaa","method":"GET","req_id":"c2ae6307-81b9-4b81-a4fa-f79e5378fe91","remote_addr":"[fd00:1122:3344:101::b]:38821","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23552,"response_code":200}
{"msg":"path is \"/data/regions/536352b3-156c-4018-875f-b07c78fab143\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.882707664Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"dataset is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.888438967Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"snapshot is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143@88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.89311233Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143\", \"88d153fb-5064-4ed9-94f3-4fdec70b8aaa\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.893144041Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 536352b3-156c-4018-875f-b07c78fab143-88d153fb-5064-4ed9-94f3-4fdec70b8aaa state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.898710261Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.904495896Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions/536352b3-156c-4018-875f-b07c78fab143/snapshots/88d153fb-5064-4ed9-94f3-4fdec70b8aaa/run","method":"POST","req_id":"10c978a1-1cf5-4b25-a37a-5505c3982396","remote_addr":"[fd00:1122:3344:101::b]:38821","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21814,"response_code":200}
{"msg":"applying SMF actions for region 536352b3-156c-4018-875f-b07c78fab143 running snapshot 88d153fb-5064-4ed9-94f3-4fdec70b8aaa (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.904515066Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-536352b3-156c-4018-875f-b07c78fab143-88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.912166388Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-536352b3-156c-4018-875f-b07c78fab143-88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.915098846Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-536352b3-156c-4018-875f-b07c78fab143-88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.92384163Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/536352b3-156c-4018-875f-b07c78fab143/.zfs/snapshot/88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.924162126Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.924204387Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.924242358Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.924278548Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.924324009Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.926135605Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.928697156Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"worker"}
{"msg":"region 536352b3-156c-4018-875f-b07c78fab143 running snapshot 88d153fb-5064-4ed9-94f3-4fdec70b8aaa state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:09.928707696Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"path is \"/data/regions/536352b3-156c-4018-875f-b07c78fab143\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:10.279130507Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"dataset is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:10.285678777Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"datafile"}
{"msg":"snapshot is oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143@88d153fb-5064-4ed9-94f3-4fdec70b8aaa","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:10.292314498Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_2e975605-afda-469b-a5a1-ee6c8d627692/crucible/regions/536352b3-156c-4018-875f-b07c78fab143\", \"88d153fb-5064-4ed9-94f3-4fdec70b8aaa\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:10.292341579Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T00:13:10.298727236Z","hostname":"oxz_crucible_031b4351-24a2-4523-ab76-815167d66a0b","pid":23965,"uri":"/crucible/0/regions/536352b3-156c-4018-875f-b07c78fab143/snapshots/88d153fb-5064-4ed9-94f3-4fdec70b8aaa/run","method":"POST","req_id":"57034ba7-c832-4b60-af4e-1a64ec48426f","remote_addr":"[fd00:1122:3344:101::b]:38821","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19642,"response_code":200}