[ Aug 31 15:43:57 Enabled. ]
[ Aug 31 15:43:57 Rereading configuration. ]
[ Aug 31 15:43:58 Rereading configuration. ]
[ Aug 31 15:44:01 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 7f9a257a-154f-434a-9798-b489dc4bee82 &"). ]
[ Aug 31 15:44:01 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:02.089843006Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:02.090674426Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"file":"oximeter/collector/src/lib.rs:299"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:02.091989797Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:02.12248176Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:12.21964348Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.219730912Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:12.219772683Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:788"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:12.220932749Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:12.221010141Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:793","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:12.223367806Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:798"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.223441847Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:12.31925581Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:819","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.356304486Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"beb1f8ec-7b26-41e1-951f-dbfecb389e7c","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.373519164Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.373545054Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.373598935Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.373608146Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:12.373653907Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.38375257Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.383929744Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.383987195Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.441243017Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.443792966Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 353.94497ms","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:12.443869058Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.798919238Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.799040081Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:12.799284116Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 548.358419ms","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:12.799336787Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:13.348400398Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:13.348499391Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:13.348676165Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 832.896175ms","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:13.348716396Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:14.18282201Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:14.182920542Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:14.295803869Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"0de52cfb-2b2d-488b-b725-2647c47b4918\", \"date\": \"Mon, 31 Aug 2026 15:44:14 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:14.295871721Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"file":"oximeter/collector/src/lib.rs:431","id":"7f9a257a-154f-434a-9798-b489dc4bee82"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:14.295925672Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:626"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:14.296959375Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:14.299724109Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:14.299949395Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-08-31T15:44:14.300540798Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:667","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.356656529Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":4}
{"msg":"unrolling 4 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.356779922Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.357798906Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":4}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.360321384Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.361998093Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_queue_depth\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:17.360281293Z }","timeseries_name":"oximeter_collector:database_queue_depth"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.362172437Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_samples_dropped\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:17.360272423Z }","timeseries_name":"oximeter_collector:database_samples_dropped"}
{"msg":"inserting 2 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.362274229Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.370596611Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.372884734Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_string","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.374738617Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u16","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.376802884Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_uuid","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.37877208Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:17.381539024Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":2}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:17.458763957Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:22.356792546Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:22.357804649Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:27.356965473Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:27.357974807Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:29.297036419Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:29.297102591Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:29.302148197Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"913b8797-aa0e-4bd4-b881-7286f832bb99\", \"content-length\": \"300\", \"date\": \"Mon, 31 Aug 2026 15:44:29 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:29.304274986Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:29.311740039Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"217fc6f2-dab1-498c-bad3-b031d10126b9\", \"content-length\": \"29\", \"date\": \"Mon, 31 Aug 2026 15:44:29 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:29.31180962Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:63195","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:29.311853841Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":1,"n_pruned_tasks":0}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:32.356535088Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:32.357744376Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:37.356808199Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:37.357840513Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:38.314909385Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:38.315009988Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:38.355926933Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140,"n_results":1}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.35711987Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140}
{"msg":"unrolling 140 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.357185051Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.357329764Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.358211325Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.363330053Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"dendrite:sample_collection_duration\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Metric, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Metric, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Metric, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Metric, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Metric, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Metric, description: \"\" }}, datum_type: F64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357312274Z }","timeseries_name":"dendrite:sample_collection_duration"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.36364568Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_data_link:link_enabled\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_id\", field_type: U8, source: Metric, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"port_id\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: Bool, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357223612Z }","timeseries_name":"switch_data_link:link_enabled"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.363828134Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_data_link:link_up\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_id\", field_type: U8, source: Metric, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"port_id\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: Bool, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357230312Z }","timeseries_name":"switch_data_link:link_up"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.363970738Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:capacity\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357236252Z }","timeseries_name":"switch_table:capacity"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364053939Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:collisions\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357257953Z }","timeseries_name":"switch_table:collisions"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364142522Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:delete_misses\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357265863Z }","timeseries_name":"switch_table:delete_misses"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364224854Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:deletes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357248562Z }","timeseries_name":"switch_table:deletes"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364315006Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:exhaustion\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357269433Z }","timeseries_name":"switch_table:exhaustion"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364384007Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:inserts\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357244692Z }","timeseries_name":"switch_table:inserts"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364594812Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:occupancy\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357240282Z }","timeseries_name":"switch_table:occupancy"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364689454Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:update_misses\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357262283Z }","timeseries_name":"switch_table:update_misses"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.364817317Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_table:updates\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"table\", field_type: String, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:42.357252193Z }","timeseries_name":"switch_table:updates"}
{"msg":"inserting 12 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.366814823Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.375185157Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_i16","n_rows":280}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.377421718Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_string","n_rows":721}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.379157038Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u16","n_rows":140}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.380911299Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u32","n_rows":145}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.382762972Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u8","n_rows":144}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.384711637Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_uuid","n_rows":420}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.386429046Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_bool","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.388102375Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":105}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.389732523Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_f64","n_rows":1}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:42.391522424Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:42.459841342Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:44.29694869Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:44.297028642Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:44.312718184Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb4748e8-4be3-4cce-b4cc-a13e1644dc9c\", \"content-length\": \"438\", \"date\": \"Mon, 31 Aug 2026 15:44:44 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:44.312804736Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:44.316277376Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8c3eed9-b130-4829-8afe-4738769a6097\", \"content-length\": \"29\", \"date\": \"Mon, 31 Aug 2026 15:44:44 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:44.316327967Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","address":"[fd00:1122:3344:101::b]:41344","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:44.316366958Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":2,"n_pruned_tasks":0}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:45.037428531Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:45.037510163Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:45.047085074Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":15,"n_results":5}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.357269647Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":15}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.357347758Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":15}
{"msg":"unrolling 15 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.35739459Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.3574353Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.3872707Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:cpus_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:47.357322578Z }","timeseries_name":"collection_target:cpus_provisioned"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.387336981Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:ram_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:47.357323708Z }","timeseries_name":"collection_target:ram_provisioned"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.387380372Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:virtual_disk_space_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:47.357324978Z }","timeseries_name":"collection_target:virtual_disk_space_provisioned"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.387421413Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"http_service:request_latency_histogram\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"operation_id\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"status_code\", field_type: U16, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-31T15:44:47.357319438Z }","timeseries_name":"http_service:request_latency_histogram"}
{"msg":"inserting 4 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.387466354Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.391157839Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_string","n_rows":6}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.393224027Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u16","n_rows":3}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.394896986Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_uuid","n_rows":15}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.397046335Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":3}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:47.398687313Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_i64","n_rows":12}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:47.459042376Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:48.315104417Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:48.31520234Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:48.326449249Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140,"n_results":1}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.356938114Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140}
{"msg":"unrolling 140 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.357025526Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.357092687Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.357994558Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.380719123Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_i16","n_rows":280}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.382605277Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_string","n_rows":721}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.384243914Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u16","n_rows":140}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.385879292Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u32","n_rows":145}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.38749816Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u8","n_rows":144}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.389354932Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_uuid","n_rows":420}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.391040662Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_bool","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.392591257Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":105}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.39443093Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_f64","n_rows":1}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:52.39617762Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:52.458356865Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:55.037374079Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:55.037471811Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:55.039373535Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":41344,"producer_ip":"fd00:1122:3344:101::b","producer_id":"2b11c91e-0206-406e-b964-68c146d5ab17","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":36,"n_results":5}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.356656931Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":36}
{"msg":"unrolling 36 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.356752304Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.356807295Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.357749317Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":36}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.361649547Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_string","n_rows":42}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.363446428Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_u16","n_rows":21}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.365145527Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"fields_uuid","n_rows":29}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.367510562Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":21}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:57.371233488Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"id":"9d2828d2-34f1-4667-b7fc-d68bb8bcc25a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","table_name":"measurements_i64","n_rows":15}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-31T15:44:57.45839208Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:58.314452541Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:58.314490882Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:58.323737685Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":63195,"producer_ip":"fd00:1122:3344:101::2","producer_id":"2e3e3296-7893-4d99-a468-6ad19656e443","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":140,"n_results":1}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:59.296493802Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.296671286Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.301468006Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7b25c47-ccce-4605-a614-f4ce50ac63d7\", \"content-length\": \"574\", \"date\": \"Mon, 31 Aug 2026 15:44:59 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.301576259Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.30423422Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/7f9a257a-154f-434a-9798-b489dc4bee82/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIyZTNlMzI5Ni03ODkzLTRkOTktYTQ2OC02YWQxOTY1NmU0NDMifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e6c6c19-207b-4340-9f34-0b6235f9d852\", \"content-length\": \"29\", \"date\": \"Mon, 31 Aug 2026 15:44:59 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.304340773Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:8001","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-31T15:44:59.304435455Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":3,"n_pruned_tasks":0}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.613448391Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.613490962Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:44:59.617554256Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:45:00.613708943Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:45:00.613789225Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-31T15:45:00.616338864Z","hostname":"oxz_oximeter_7f9a257a-154f-434a-9798-b489dc4bee82","pid":23169,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"288b5986-7cb7-4ab6-8169-92403ca358df","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"7f9a257a-154f-434a-9798-b489dc4bee82","component":"oximeter-agent","n_samples":26,"n_results":1}