[ Jul  2 18:44:42 Enabled. ]
[ Jul  2 18:44:42 Rereading configuration. ]
[ Jul  2 18:44:43 Rereading configuration. ]
[ Jul  2 18:44:46 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 71014f97-c82d-441f-9f05-0bca871524fa &"). ]
[ Jul  2 18:44:46 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:46.772244217Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:46.790724387Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:301"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:46.79153952Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:46.798306046Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:51.732850401Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.732904632Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:51.732917202Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:51.733926099Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:51.733976163Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:51.733987094Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.733996019Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:51.823649376Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"2253a2b6-3065-4931-92ce-8fa66defb041","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.841572559Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"id":"68be1fba-da2e-4efc-aef1-0720fa1c3ab6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.855708849Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.855741629Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.855754109Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.85576001Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:51.855767919Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.858411821Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.858487742Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.858500353Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.875903366Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:51.876583506Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 168.873661ms","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:51.876615247Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.046751956Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.046830287Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.047082521Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 658.149957ms","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:52.047100521Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.706329424Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.706430545Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:52.70670259Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.069952254s","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:52.70671869Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:53.777914718Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:53.77803503Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:53.778200172Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.600685882s","v":0,"name":"oximeter","level":40,"time":"2026-07-02T18:44:53.778229253Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.379274285Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.379373287Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.418751125Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"8e24909d-4bcd-4fba-a10a-1f5cc568395a\", \"date\": \"Thu, 02 Jul 2026 18:44:56 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:56.42357166Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"file":"oximeter/collector/src/lib.rs:464","id":"71014f97-c82d-441f-9f05-0bca871524fa"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:56.423647051Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:562"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:56.424701998Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.427818637Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/71014f97-c82d-441f-9f05-0bca871524fa/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.455077805Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/71014f97-c82d-441f-9f05-0bca871524fa/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63c53b28-e231-49c0-ad13-a4ac3aa0be91\", \"content-length\": \"29\", \"date\": \"Thu, 02 Jul 2026 18:44:56 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-02T18:44:56.455188076Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.842824679Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-02T18:44:56.84291295Z","hostname":"oxz_oximeter_71014f97-c82d-441f-9f05-0bca871524fa","pid":22620,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"71014f97-c82d-441f-9f05-0bca871524fa","component":"oximeter-agent"}