[ May 14 16:25:46 Enabled. ]
[ May 14 16:25:46 Rereading configuration. ]
[ May 14 16:25:48 Rereading configuration. ]
[ May 14 16:25:51 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.535357349Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146}
{"msg":"dataset: \"oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.536180277Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.536197318Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.536204698Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.555640253Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:25:51.620289486Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:58.983914072Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56075"}
{"msg":"region 95459e73-1547-40f3-9d7a-7e27c326a939 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.021356148Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.022963855Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.022998255Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions","method":"POST","req_id":"052dbd27-b497-434c-a5cd-bea25560dafb","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":15463,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible/regions/95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.031607916Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible/regions/95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.031629506Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"95459e73-1547-40f3-9d7a-7e27c326a939\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/95459e73-1547-40f3-9d7a-7e27c326a939\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.083884708Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"region":"95459e73-1547-40f3-9d7a-7e27c326a939","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"95459e73-1547-40f3-9d7a-7e27c326a939\" \"--data\" \"/data/regions/95459e73-1547-40f3-9d7a-7e27c326a939\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.086040931Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"region":"95459e73-1547-40f3-9d7a-7e27c326a939","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.209051969Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"region":"95459e73-1547-40f3-9d7a-7e27c326a939","component":"worker"}
{"msg":"region 95459e73-1547-40f3-9d7a-7e27c326a939 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.20910641Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.209198061Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.209521765Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.215638009Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.226766857Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.228200512Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.229071851Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.229157632Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.229221073Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.229263953Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.231432056Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.23369286Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.373963121Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions","method":"POST","req_id":"029a2b59-d11b-497d-972d-3f5a921aa5b6","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.637172029Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions/95459e73-1547-40f3-9d7a-7e27c326a939","method":"GET","req_id":"b6009039-dcbb-4e27-9f00-e0e90e3591bb","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"path is \"/data/regions/95459e73-1547-40f3-9d7a-7e27c326a939\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.64007845Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"dataset is oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible/regions/95459e73-1547-40f3-9d7a-7e27c326a939","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.648545919Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"region 95459e73-1547-40f3-9d7a-7e27c326a939 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.655585553Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.655738635Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.655754905Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions/95459e73-1547-40f3-9d7a-7e27c326a939","method":"DELETE","req_id":"4c3a0b8b-a9c8-446d-a3c4-0541c65f79f4","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":15721,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-95459e73-1547-40f3-9d7a-7e27c326a939 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.656876637Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.658899538Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions/95459e73-1547-40f3-9d7a-7e27c326a939","method":"GET","req_id":"63a145bf-34ea-4a6f-923d-bd67642ab4a3","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":36,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.659800908Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9fdfd14a-6d44-4c8e-bb64-2fb8e642f7b5/crucible/regions/95459e73-1547-40f3-9d7a-7e27c326a939\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.66762797Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"region":"95459e73-1547-40f3-9d7a-7e27c326a939","component":"worker"}
{"msg":"region 95459e73-1547-40f3-9d7a-7e27c326a939 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.720455468Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-14T16:26:59.760750424Z","hostname":"oxz_crucible_26671b00-c712-48c3-9200-8a3c9e183d38","pid":23146,"uri":"/crucible/0/regions/95459e73-1547-40f3-9d7a-7e27c326a939","method":"GET","req_id":"7a9d1cf4-1995-4b13-955d-8a4f48f8bb44","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":56,"response_code":200}