[ May 29 00:01:50 Enabled. ]
[ May 29 00:01:50 Rereading configuration. ]
[ May 29 00:01:53 Rereading configuration. ]
[ May 29 00:01:59 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:01:59.967542371Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511}
{"msg":"dataset: \"oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:01:59.967751494Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:01:59.967762745Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:01:59.967769025Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:01:59.978518629Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:02:00.127064002Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.464393994Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57551"}
{"msg":"region bb6dee71-2967-46db-a4ea-3dc5ecebfe35 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.491551258Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.497560149Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.49758606Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions","method":"POST","req_id":"b2c47049-eb1c-499e-89a3-4323c7577026","remote_addr":"[fd00:1122:3344:101::b]:57551","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16064,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.508388054Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.508416595Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bb6dee71-2967-46db-a4ea-3dc5ecebfe35\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.555900389Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"bb6dee71-2967-46db-a4ea-3dc5ecebfe35","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bb6dee71-2967-46db-a4ea-3dc5ecebfe35\" \"--data\" \"/data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.55795824Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"bb6dee71-2967-46db-a4ea-3dc5ecebfe35","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.721809356Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions","method":"POST","req_id":"61896b12-f530-41e7-85b9-59fd3bb8ba00","remote_addr":"[fd00:1122:3344:101::b]:57551","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":84,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.945434493Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"bb6dee71-2967-46db-a4ea-3dc5ecebfe35","component":"worker"}
{"msg":"region bb6dee71-2967-46db-a4ea-3dc5ecebfe35 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.945482004Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.945604666Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.947953762Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.953548127Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.968384943Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.972514396Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.973187806Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.973249037Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.973289408Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.973325578Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.984647151Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:19.98918239Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:03:20.322790113Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions","method":"POST","req_id":"5e87e312-153b-4d06-93f6-35bccdc661a5","remote_addr":"[fd00:1122:3344:101::b]:57551","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.680986635Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62524"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.681214879Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","method":"GET","req_id":"f729d97c-0fdb-4429-8c8c-f7b2eb32f340","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"path is \"/data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.684072212Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"dataset is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.690217796Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"snapshot is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35@55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.695650039Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\", \"55b5b25d-9327-4292-90a6-2a90bacdf734\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.695679169Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.704379672Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35/snapshots/55b5b25d-9327-4292-90a6-2a90bacdf734","method":"GET","req_id":"3886b2c0-88d3-42ff-ba61-cbd5d78d2d31","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20336,"response_code":200}
{"msg":"path is \"/data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.70823664Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"dataset is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.714713639Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"snapshot is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35@55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.720045831Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\", \"55b5b25d-9327-4292-90a6-2a90bacdf734\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.720063731Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot bb6dee71-2967-46db-a4ea-3dc5ecebfe35-55b5b25d-9327-4292-90a6-2a90bacdf734 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.727961431Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"applying SMF actions for region bb6dee71-2967-46db-a4ea-3dc5ecebfe35 running snapshot 55b5b25d-9327-4292-90a6-2a90bacdf734 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.730957817Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.730973487Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35/snapshots/55b5b25d-9327-4292-90a6-2a90bacdf734/run","method":"POST","req_id":"a549e963-5e14-4756-a442-6fd587c64813","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22732,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746536144Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746614775Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746687086Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746694557Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746730907Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.746775188Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.747146033Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.747197004Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.747239075Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.747279975Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.754965632Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-bb6dee71-2967-46db-a4ea-3dc5ecebfe35-55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.757599963Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-bb6dee71-2967-46db-a4ea-3dc5ecebfe35-55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.765817468Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-bb6dee71-2967-46db-a4ea-3dc5ecebfe35-55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778270578Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35/.zfs/snapshot/55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778528282Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778613333Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778669224Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778721854Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.778778295Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.783340245Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.787129193Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"region bb6dee71-2967-46db-a4ea-3dc5ecebfe35 running snapshot 55b5b25d-9327-4292-90a6-2a90bacdf734 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:13.787145463Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"path is \"/data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:14.00220697Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"dataset is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:14.010560227Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"snapshot is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35@55b5b25d-9327-4292-90a6-2a90bacdf734","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:14.017278469Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35\", \"55b5b25d-9327-4292-90a6-2a90bacdf734\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:14.01730472Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:14.023259211Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35/snapshots/55b5b25d-9327-4292-90a6-2a90bacdf734/run","method":"POST","req_id":"9f6f5fdb-e4bb-49a8-b631-8a6d70de50ff","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21092,"response_code":200}
{"msg":"region a1923078-c742-4779-bbd1-4012f2f6d4bf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.430507433Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.430712576Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.430731026Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions","method":"POST","req_id":"e8234850-eee4-4f68-a644-10a931eeec1b","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":264,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.437865395Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.437897986Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a1923078-c742-4779-bbd1-4012f2f6d4bf\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.484220062Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"a1923078-c742-4779-bbd1-4012f2f6d4bf","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a1923078-c742-4779-bbd1-4012f2f6d4bf\" \"--data\" \"/data/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.484263952Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"a1923078-c742-4779-bbd1-4012f2f6d4bf","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.56946239Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"a1923078-c742-4779-bbd1-4012f2f6d4bf","component":"worker"}
{"msg":"region a1923078-c742-4779-bbd1-4012f2f6d4bf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.569490041Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.569598652Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.569940157Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.575475282Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.582990716Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.586347838Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.586641872Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.586683813Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.586724263Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.586757514Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.590665213Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.595950515Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.596022694Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.596095454Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.596104015Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.59614039Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.596182897Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.600751977Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.600804658Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.600844959Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.600880919Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.604766228Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.606554466Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:18.760065944Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions","method":"POST","req_id":"d5416ac5-07de-4086-bf8c-abfb9c617be9","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.856722264Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","method":"GET","req_id":"57247c64-dfdb-4f7d-a93b-0a5594855a98","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"path is \"/data/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.859881602Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"dataset is oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.871169854Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"region a1923078-c742-4779-bbd1-4012f2f6d4bf state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.878445415Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.879112295Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.879133696Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","method":"DELETE","req_id":"1feb1b1f-7cd1-4d49-8818-815b46fa7ca2","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19263,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-a1923078-c742-4779-bbd1-4012f2f6d4bf (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.88008118Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.882305774Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","method":"GET","req_id":"d0d26763-2f5d-4e28-aa45-a7352772d0b5","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":44,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889614405Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889692706Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889774178Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889789318Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889823739Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.889883439Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bb6dee71-2967-46db-a4ea-3dc5ecebfe35","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.890257885Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.890331466Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.890406907Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.890464458Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.89455114Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.895912971Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1f553004-10ef-4473-a6a7-4a0bae461219/crucible/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.904144407Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"region":"a1923078-c742-4779-bbd1-4012f2f6d4bf","component":"worker"}
{"msg":"region a1923078-c742-4779-bbd1-4012f2f6d4bf state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:44.957372198Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-29T00:14:45.024434019Z","hostname":"oxz_crucible_e2f41577-7725-4420-83ee-24d90ea1a803","pid":26511,"uri":"/crucible/0/regions/a1923078-c742-4779-bbd1-4012f2f6d4bf","method":"GET","req_id":"d8f33665-df7f-413e-8e8c-d1cfd15b1f9c","remote_addr":"[fd00:1122:3344:101::b]:62524","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":200}