[ Jul 22 07:46:26 Enabled. ]
[ Jul 22 07:46:26 Rereading configuration. ]
[ Jul 22 07:46:27 Rereading configuration. ]
[ Jul 22 07:46:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.015015983Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421}
{"msg":"dataset: \"oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.015263988Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.015277518Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.015284518Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.032472704Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:46:32.265281349Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.937841647Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42133"}
{"msg":"region 94934962-041a-48ee-9234-d0cebcf5558a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.968913199Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.973350781Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.975463465Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions","method":"POST","req_id":"a83c415b-ef9c-4d67-8c4a-a4f588c81d78","remote_addr":"[fd00:1122:3344:101::c]:42133","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":15425,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.981695204Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:17.981710514Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"94934962-041a-48ee-9234-d0cebcf5558a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/94934962-041a-48ee-9234-d0cebcf5558a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.056987441Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"94934962-041a-48ee-9234-d0cebcf5558a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"94934962-041a-48ee-9234-d0cebcf5558a\" \"--data\" \"/data/regions/94934962-041a-48ee-9234-d0cebcf5558a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.059141666Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"94934962-041a-48ee-9234-d0cebcf5558a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.118073515Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions","method":"POST","req_id":"460c1bab-ff33-4b10-bffa-2d45dfb0b24d","remote_addr":"[fd00:1122:3344:101::c]:42133","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.226433876Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"94934962-041a-48ee-9234-d0cebcf5558a","component":"worker"}
{"msg":"region 94934962-041a-48ee-9234-d0cebcf5558a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.23044413Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.230716165Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.233023513Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.240881535Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.25414139Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.257644722Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.258368557Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.260368138Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.26041409Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.260473301Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.271373016Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.275833278Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:47:18.613758199Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions","method":"POST","req_id":"06389ca0-7e7f-4997-8935-c65dc5e2bd7c","remote_addr":"[fd00:1122:3344:101::c]:42133","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":78,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.911470617Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44167"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.911584409Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/94934962-041a-48ee-9234-d0cebcf5558a","method":"GET","req_id":"465017a2-09b2-403e-8304-dd039349c677","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":42,"response_code":200}
{"msg":"path is \"/data/regions/94934962-041a-48ee-9234-d0cebcf5558a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.915783036Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"dataset is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.921929203Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"snapshot is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a@4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.927275363Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a\", \"4b057a89-3cff-48e4-9a10-3bbaacf73209\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.927305134Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.937655078Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/94934962-041a-48ee-9234-d0cebcf5558a/snapshots/4b057a89-3cff-48e4-9a10-3bbaacf73209","method":"GET","req_id":"fc5c8985-7a44-4b53-ad83-131dcfd29a36","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21885,"response_code":200}
{"msg":"path is \"/data/regions/94934962-041a-48ee-9234-d0cebcf5558a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.940868325Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"dataset is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.945510121Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"snapshot is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a@4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.950221938Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a\", \"4b057a89-3cff-48e4-9a10-3bbaacf73209\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.950236509Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 94934962-041a-48ee-9234-d0cebcf5558a-4b057a89-3cff-48e4-9a10-3bbaacf73209 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.957819455Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"applying SMF actions for region 94934962-041a-48ee-9234-d0cebcf5558a running snapshot 4b057a89-3cff-48e4-9a10-3bbaacf73209 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.960907839Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.96092581Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/94934962-041a-48ee-9234-d0cebcf5558a/snapshots/4b057a89-3cff-48e4-9a10-3bbaacf73209/run","method":"POST","req_id":"4c19f30d-eb48-466a-a711-e566a758d021","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20079,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.96724308Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967335002Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967417794Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967424594Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967461905Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967506356Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967601338Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.967651129Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.96769494Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.9677363Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.975553262Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-94934962-041a-48ee-9234-d0cebcf5558a-4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.978114265Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-94934962-041a-48ee-9234-d0cebcf5558a-4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.981367582Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-94934962-041a-48ee-9234-d0cebcf5558a-4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.993622346Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94934962-041a-48ee-9234-d0cebcf5558a/.zfs/snapshot/4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.993923022Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.993989524Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.994013294Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.994053535Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.994099116Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:20.997521807Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.001907007Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"region 94934962-041a-48ee-9234-d0cebcf5558a running snapshot 4b057a89-3cff-48e4-9a10-3bbaacf73209 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.001916288Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"path is \"/data/regions/94934962-041a-48ee-9234-d0cebcf5558a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.253157615Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"dataset is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.260254742Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"snapshot is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a@4b057a89-3cff-48e4-9a10-3bbaacf73209","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.266555492Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/94934962-041a-48ee-9234-d0cebcf5558a\", \"4b057a89-3cff-48e4-9a10-3bbaacf73209\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.266593453Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:21.271509955Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/94934962-041a-48ee-9234-d0cebcf5558a/snapshots/4b057a89-3cff-48e4-9a10-3bbaacf73209/run","method":"POST","req_id":"b9956867-4d2f-4fcd-bace-14b1f47fbb2c","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18407,"response_code":200}
{"msg":"region b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.568364138Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.568550662Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.568569432Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions","method":"POST","req_id":"e1ff84d9-b2ce-4f85-9e61-97d1aca0ba57","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":233,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.57522182Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.57524849Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.62210211Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\" \"--data\" \"/data/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.62214529Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.696254873Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","component":"worker"}
{"msg":"region b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.696282894Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.696402867Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.696905017Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.696981348Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.69706055Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697084161Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697091141Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697150592Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697514639Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.69755026Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697599551Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.697640822Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.703712518Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.704104856Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.709748503Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.718194777Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.727087401Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.727414848Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.727487899Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.727552791Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.727590612Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.731635175Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.731805779Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions","method":"POST","req_id":"abae90af-8232-4ffd-bd81-d61838736c4b","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:25.737760212Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.77853071Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","method":"GET","req_id":"f6f5bdc5-4144-4e7f-84af-31e1d53d2c12","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"path is \"/data/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.782283177Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"dataset is oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.792689402Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"region b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.800178727Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.800349401Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.802778141Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","method":"DELETE","req_id":"a1356e04-7c58-4593-b6d2-346178f2a292","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20524,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.803644729Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.805952127Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","method":"GET","req_id":"ee8c56c9-d5bd-4c7a-aa81-dfc93739aa0b","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":35,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808091451Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808213393Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808299995Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808318356Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808346336Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808383197Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94934962-041a-48ee-9234-d0cebcf5558a","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808694214Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808762785Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808822186Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.808859827Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.812619385Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.813880781Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c0b050bf-d482-4d23-bf38-c836e65b5ee7/crucible/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.822110401Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"region":"b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","component":"worker"}
{"msg":"region b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.870340919Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T07:58:50.878745392Z","hostname":"oxz_crucible_4f169622-0968-4338-bd68-fdf29523c7bd","pid":27421,"uri":"/crucible/0/regions/b5d7cc0c-aef5-4c64-bc0c-79ad6f70af69","method":"GET","req_id":"59d8e5f0-2a71-4c11-b1f9-4bb4be882a19","remote_addr":"[fd00:1122:3344:101::c]:44167","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":39,"response_code":200}