[ Aug 29 02:59:23 Enabled. ] [ Aug 29 02:59:23 Rereading configuration. ] [ Aug 29 02:59:24 Rereading configuration. ] [ Aug 29 02:59:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id f38e6b55-3437-427d-ba44-a007db9943fd &"). ] [ Aug 29 02:59:28 Method "start" exited with status 0. ] note: configured to log to "/dev/stdout" {"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:28.694294131Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:28.715885783Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:299"} {"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:28.716818959Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:28.726563702Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:34.082297917Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"} {"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.082360998Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:34.082372789Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:788"} {"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:34.083594489Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"} {"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:34.083618879Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:793","version":0} {"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:34.083628599Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:798"} {"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.08363688Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:34.180092396Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:819","version":14} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.206234124Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"7e882303-b208-4f96-8e7b-08b30e18a4e3","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.21553169Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.21556435Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.215580031Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.215586981Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:34.215597831Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.219599418Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.21970132Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.2197145Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.238028087Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.239518012Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 279.654564ms","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:34.239541262Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.520675723Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.520757295Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:34.520931988Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 714.458462ms","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:34.520946148Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:35.237127768Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:35.23723923Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:35.237558846Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 1.211260058s","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:35.237587776Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:36.449764508Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:36.44987847Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:36.450129385Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 2.522215844s","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:36.450200006Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:38.973667921Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:38.973761903Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:38.973940476Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 4.288567273s","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:38.973991026Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.208092829Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":4} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.208158401Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":4} {"msg":"unrolling 4 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.208205782Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.208998075Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.210899376Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_queue_depth\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:39.208945154Z }","timeseries_name":"oximeter_collector:database_queue_depth"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.211029049Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_samples_dropped\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:39.208939854Z }","timeseries_name":"oximeter_collector:database_samples_dropped"} {"msg":"inserting 2 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.21113616Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.218975962Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.221177449Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_string","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.223498128Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_u16","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.227167519Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_uuid","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.229407377Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:39.232675061Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":2} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:39.310573227Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:43.263994364Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:43.264105516Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:43.293030951Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"623ee10a-b7e3-42ab-9d79-1448c88dadf8\", \"date\": \"Sat, 29 Aug 2026 02:59:43 GMT\"} })"} {"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:43.293063001Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"file":"oximeter/collector/src/lib.rs:431","id":"f38e6b55-3437-427d-ba44-a007db9943fd"} {"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:43.293136653Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:626"} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:43.2941759Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:43.295222737Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:43.315321374Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17be06d9-d6c3-4195-b923-eca2f0117f79\", \"content-length\": \"29\", \"date\": \"Sat, 29 Aug 2026 02:59:43 GMT\"} })"} {"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:43.317658293Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":0,"n_pruned_tasks":0} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:44.207735999Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:44.238985102Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:49.207372067Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:49.207461169Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:54.207702277Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:54.207779509Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:58.294831025Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.294907826Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.30169497Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe19f803-d99e-4380-8aa7-450370fe9115\", \"content-length\": \"713\", \"date\": \"Sat, 29 Aug 2026 02:59:58 GMT\"} })"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.301765511Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhZDdkODc0MS0xYTU2LTQyNjktYjY4MC0zMzRjNmZhYjE0OGYifX0%3D","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.304804072Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/f38e6b55-3437-427d-ba44-a007db9943fd/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhZDdkODc0MS0xYTU2LTQyNjktYjY4MC0zMzRjNmZhYjE0OGYifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca5e4b72-98e7-40b4-8db9-3acf7ed37c41\", \"content-length\": \"29\", \"date\": \"Sat, 29 Aug 2026 02:59:58 GMT\"} })"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.304874333Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:46317","producer_id":"1bdc9ef8-7f0b-4956-acbf-14f0887928fc"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.304917494Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:8001","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.304971195Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:46935","producer_id":"9feed2e5-3b25-47cb-9d39-7ba5e243fc18"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.304982715Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:4677","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f"} {"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-29T02:59:58.305038736Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":4,"n_pruned_tasks":0} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.740646645Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.740728236Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:58.759907868Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":4,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.017037377Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.017116698Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.019402406Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.207676821Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":30} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.207706992Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":30} {"msg":"unrolling 30 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.207753902Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.207828814Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.213825054Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207756793Z }","timeseries_name":"ddm_router:originated_tunnel_endpoints"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.213911906Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207755002Z }","timeseries_name":"ddm_router:originated_underlay_prefixes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.213972467Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207763873Z }","timeseries_name":"ddm_session:advertisements_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214127619Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207762123Z }","timeseries_name":"ddm_session:advertisements_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214210571Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207791873Z }","timeseries_name":"ddm_session:imported_tunnel_endpoints"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214261842Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207790163Z }","timeseries_name":"ddm_session:imported_underlay_prefixes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214315292Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_address_changes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207780813Z }","timeseries_name":"ddm_session:peer_address_changes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214359873Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_expirations\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207765593Z }","timeseries_name":"ddm_session:peer_expirations"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214411724Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_sessions_established\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207783443Z }","timeseries_name":"ddm_session:peer_sessions_established"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214465205Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207760353Z }","timeseries_name":"ddm_session:solicitations_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214544726Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207758693Z }","timeseries_name":"ddm_session:solicitations_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214620288Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:update_send_fail\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207788373Z }","timeseries_name":"ddm_session:update_send_fail"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214683649Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207786493Z }","timeseries_name":"ddm_session:updates_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.21473567Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207785063Z }","timeseries_name":"ddm_session:updates_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.21479384Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"mg_lower:routes_blocked_by_link_state\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207752382Z }","timeseries_name":"mg_lower:routes_blocked_by_link_state"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214851352Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"static_routing_config:static_nexthops\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207747682Z }","timeseries_name":"static_routing_config:static_nexthops"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214908173Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"static_routing_config:static_routes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207744342Z }","timeseries_name":"static_routing_config:static_routes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214949513Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_rib:active_routes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-29T02:59:59.207750453Z }","timeseries_name":"switch_rib:active_routes"} {"msg":"inserting 18 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.214993394Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.223198951Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_string","n_rows":54} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.225434049Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"fields_uuid","n_rows":60} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.2272784Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":22} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.228976288Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"id":"c2229877-ee40-47d9-822d-af3156608bea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","table_name":"measurements_u64","n_rows":8} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-29T02:59:59.309182642Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.740250125Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.740324446Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T02:59:59.741038169Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":4,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.017165766Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.017242607Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.019045897Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"95d5aac0-9c1d-4ac1-90d7-eed99040ef86","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.740257692Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.740292103Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-29T03:00:00.741260739Z","hostname":"oxz_oximeter_f38e6b55-3437-427d-ba44-a007db9943fd","pid":22745,"producer_port":4677,"producer_ip":"fd00:1122:3344:101::2","producer_id":"ad7d8741-1a56-4269-b680-334c6fab148f","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"f38e6b55-3437-427d-ba44-a007db9943fd","component":"oximeter-agent","n_samples":4,"n_results":1}