[ Mar 12 01:34:39 Enabled. ]
[ Mar 12 01:34:39 Rereading configuration. ]
[ Mar 12 01:34:41 Rereading configuration. ]
[ Mar 12 01:34:47 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.777752591Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546}
{"msg":"dataset: \"oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.778542893Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.778553033Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.778558233Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.791765904Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:34:47.980070984Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.951134699Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45725"}
{"msg":"region 3c31dff9-04d5-435a-8132-b62755ba063d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.984560175Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.987749153Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.987800134Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"7e3ff59d-3664-40e3-94ab-9b2e591b5cc7","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":17104,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible/regions/3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.997078094Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible/regions/3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:10.997091634Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3c31dff9-04d5-435a-8132-b62755ba063d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3c31dff9-04d5-435a-8132-b62755ba063d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.078794411Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"region":"3c31dff9-04d5-435a-8132-b62755ba063d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3c31dff9-04d5-435a-8132-b62755ba063d\" \"--data\" \"/data/regions/3c31dff9-04d5-435a-8132-b62755ba063d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.081209728Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"region":"3c31dff9-04d5-435a-8132-b62755ba063d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.364483255Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"e40c8cd9-df4f-423e-bdfe-49e0ec5c9347","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.421990626Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"region":"3c31dff9-04d5-435a-8132-b62755ba063d","component":"worker"}
{"msg":"region 3c31dff9-04d5-435a-8132-b62755ba063d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.422066467Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.42226591Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.422593825Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.42881544Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.443330729Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.447625334Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.448359895Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.448425706Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.448474217Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.448512798Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.452875454Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.457480783Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:11.657814976Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions","method":"POST","req_id":"7cd7135e-de37-4b0d-8643-1acd8c8335a5","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.133393136Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions/3c31dff9-04d5-435a-8132-b62755ba063d","method":"GET","req_id":"16928335-3ea0-40b1-ae17-02f4c1f32bb5","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":90,"response_code":200}
{"msg":"path is \"/data/regions/3c31dff9-04d5-435a-8132-b62755ba063d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.137215613Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"dataset is oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible/regions/3c31dff9-04d5-435a-8132-b62755ba063d","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.145751393Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"region 3c31dff9-04d5-435a-8132-b62755ba063d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.153080164Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.153259016Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.153302607Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions/3c31dff9-04d5-435a-8132-b62755ba063d","method":"DELETE","req_id":"30fb9dab-e177-4b45-85f9-ed17f0249ab6","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16069,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-3c31dff9-04d5-435a-8132-b62755ba063d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.154254111Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.156662437Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions/3c31dff9-04d5-435a-8132-b62755ba063d","method":"GET","req_id":"e29115b3-8234-44a6-965f-b5e8dcd73158","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":40,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.159191476Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_5c689741-40f7-4604-8448-e2e779b399e6/crucible/regions/3c31dff9-04d5-435a-8132-b62755ba063d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.167826156Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"region":"3c31dff9-04d5-435a-8132-b62755ba063d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.223727153Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions/3c31dff9-04d5-435a-8132-b62755ba063d","method":"GET","req_id":"a0429c48-c0ca-414a-864e-46f4e671f727","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region 3c31dff9-04d5-435a-8132-b62755ba063d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.250072341Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-12T01:35:12.365867052Z","hostname":"oxz_crucible_54339118-6a59-4b20-a40c-cae009922a32","pid":25546,"uri":"/crucible/0/regions/3c31dff9-04d5-435a-8132-b62755ba063d","method":"GET","req_id":"877955c3-a2c4-4b29-9e3d-41ce6123a1f6","remote_addr":"[fd00:1122:3344:101::a]:45725","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":53,"response_code":200}