[ Jun 24 04:43:04 Enabled. ] [ Jun 24 04:43:04 Rereading configuration. ] [ Jun 24 04:43:06 Rereading configuration. ] [ Jun 24 04:43:11 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 3644ccd5-3d06-4276-907c-a58b9b7d5591 &"). ] [ Jun 24 04:43:11 Method "start" exited with status 0. ] note: configured to log to "/dev/stdout" {"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:11.827463381Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:11.834892983Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:301"} {"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:11.835687921Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:11.841415784Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:13.844318829Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"} {"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:13.84438416Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:13.8444012Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"} {"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:13.845328262Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"} {"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:13.845346442Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0} {"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:13.845358013Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"} {"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:13.845366723Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:14.729830519Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.747516889Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"6b5ae4f6-b2d9-4241-b070-ebb7ed16fd66","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.756673952Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.756699762Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.756712682Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.756719022Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:14.756727033Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.758940054Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"} {"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.759020946Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.759036176Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.782632163Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:14.786916492Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 333.26092ms","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:14.786951853Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.121034855Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.121117177Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.121344152Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 733.413849ms","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:15.121392513Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.855468695Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.855578697Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:15.855742131Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 699.591906ms","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:15.855755591Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:16.556822018Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:16.55691979Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:16.557104974Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 1.302648119s","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:16.557122035Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:17.861237305Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:17.861325247Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:17.861523432Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to contact nexus, will retry in 4.644527853s","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:17.861570363Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:19.764573473Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:19.764653425Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:22.507182709Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:22.507269271Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:22.839990661Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"5adc7c06-8cb9-498b-bf54-a7d0894ae9ff\", \"date\": \"Wed, 24 Jun 2026 04:43:22 GMT\"} })"} {"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:22.840114964Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"file":"oximeter/collector/src/lib.rs:464","id":"3644ccd5-3d06-4276-907c-a58b9b7d5591"} {"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:22.840242727Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:562"} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:22.841301442Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:22.845204732Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:22.886008758Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"142d3f8c-7c29-4027-8523-41d6df8b2ffb\", \"content-length\": \"29\", \"date\": \"Wed, 24 Jun 2026 04:43:22 GMT\"} })"} {"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:22.886153251Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":0,"n_pruned_tasks":0} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:24.765382739Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:24.765493712Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:29.764845843Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:29.764886853Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:34.764967902Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:34.765016183Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:37.841239377Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.841300838Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.981597859Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09cd88d3-cd23-4e42-b141-a16a785d9a48\", \"content-length\": \"436\", \"date\": \"Wed, 24 Jun 2026 04:43:37 GMT\"} })"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.986541284Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhODJmMjViYi0xZGUyLTQ1YTgtYjIyMi1iYmM5NTljZTgwY2EifX0%3D","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.992386709Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhODJmMjViYi0xZGUyLTQ1YTgtYjIyMi1iYmM5NTljZTgwY2EifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c2f65ab-408b-4d48-a65b-4d67cd574c02\", \"content-length\": \"29\", \"date\": \"Wed, 24 Jun 2026 04:43:37 GMT\"} })"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.99242177Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:8001","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160"} {"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:37.997523478Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:35101","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca"} {"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:37.997559019Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":2,"n_pruned_tasks":0} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.000069767Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.000102868Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.000117468Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.000127688Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.021740859Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.02610064Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.040521254Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.065281548Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":264,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.99882167Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:38.998903272Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.00208906Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.765358284Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":318} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.765399275Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":318} {"msg":"unrolling 318 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.76733157Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.786200807Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804300446Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770243617Z }","timeseries_name":"ddm_router:originated_tunnel_endpoints"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804353237Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770175515Z }","timeseries_name":"ddm_router:originated_underlay_prefixes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804369698Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770592595Z }","timeseries_name":"ddm_session:advertisements_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804390418Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770477562Z }","timeseries_name":"ddm_session:advertisements_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804410779Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771091546Z }","timeseries_name":"ddm_session:imported_tunnel_endpoints"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804431199Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771029975Z }","timeseries_name":"ddm_session:imported_underlay_prefixes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.80445025Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_address_changes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770722548Z }","timeseries_name":"ddm_session:peer_address_changes"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.80447059Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_expirations\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770657327Z }","timeseries_name":"ddm_session:peer_expirations"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804493331Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_sessions_established\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770786730Z }","timeseries_name":"ddm_session:peer_sessions_established"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804517131Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770381350Z }","timeseries_name":"ddm_session:solicitations_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804538882Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770319819Z }","timeseries_name":"ddm_session:solicitations_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804559432Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:update_send_fail\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770968864Z }","timeseries_name":"ddm_session:update_send_fail"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804576442Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770906802Z }","timeseries_name":"ddm_session:updates_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804593153Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770846751Z }","timeseries_name":"ddm_session:updates_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804611113Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:collections\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"base_route\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"producer_id\", field_type: Uuid, source: Metric, description: \"\" }, FieldSchema { name: \"producer_ip\", field_type: IpAddr, source: Metric, description: \"\" }, FieldSchema { name: \"producer_port\", field_type: U16, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.770032352Z }","timeseries_name":"oximeter_collector:collections"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804635614Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:bytes_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771640759Z }","timeseries_name":"switch_port_control_data_link:bytes_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804681635Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:bytes_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771812153Z }","timeseries_name":"switch_port_control_data_link:bytes_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804721426Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:errors_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771508376Z }","timeseries_name":"switch_port_control_data_link:errors_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804756377Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:errors_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771575668Z }","timeseries_name":"switch_port_control_data_link:errors_sent"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804793727Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:packets_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771728391Z }","timeseries_name":"switch_port_control_data_link:packets_received"} {"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.804830188Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"switch_port_control_data_link:packets_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"asic_fab\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_lot\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer\", field_type: U8, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_x\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"asic_wafer_loc_y\", field_type: I16, source: Target, description: \"\" }, FieldSchema { name: \"kind\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"link_name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"network\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"sled_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"sled_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"switch_model\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_revision\", field_type: U32, source: Target, description: \"\" }, FieldSchema { name: \"switch_serial\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"switch_slot\", field_type: U16, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-06-24T04:43:39.771893145Z }","timeseries_name":"switch_port_control_data_link:packets_sent"} {"msg":"inserting 21 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.81394988Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.838762975Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_i16","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.840381322Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.841772795Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":172} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.843091685Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u16","n_rows":16} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.844451537Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u32","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.845751367Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u8","n_rows":12} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.847026296Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":92} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.852707408Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":306} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.854020548Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":12} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:39.868410672Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.9982394Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:39.998281411Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:40.000630726Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:40.998829157Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:40.998877498Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:41.001179942Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:41.99798677Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:41.998039001Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:42.0005591Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:42.9987038Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:42.998746021Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:43.001080645Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:43.998367215Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:43.998413345Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.000726629Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.76589153Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.765934891Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.765948132Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.770444496Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.819002801Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.820502106Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.821734144Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.822860211Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:44.867028914Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.998108932Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:44.998155033Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:45.000522528Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:45.998697558Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:45.9987743Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:46.002310252Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:46.998566349Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:46.998640141Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:47.001372914Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:48.355773985Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:48.355820666Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:48.358098558Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:49.356295119Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:49.35634826Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:49.358696745Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.122880313Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.122917214Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.122940304Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.127463399Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.131986464Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.133134111Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.134327228Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.135435724Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:50.223787401Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.355861772Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.355914333Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:50.358306019Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:51.355496717Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:51.355563999Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:51.35820138Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:52.355878418Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:52.355930209Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:52.358327725Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:53.198990806Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.199046187Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.264487463Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"043caf9f-0df1-41d2-bdb0-a27e94ce0895\", \"content-length\": \"436\", \"date\": \"Wed, 24 Jun 2026 04:43:53 GMT\"} })"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.264529555Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhODJmMjViYi0xZGUyLTQ1YTgtYjIyMi1iYmM5NTljZTgwY2EifX0%3D","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.271319242Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJhODJmMjViYi0xZGUyLTQ1YTgtYjIyMi1iYmM5NTljZTgwY2EifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8cef3847-34a0-4672-9a1c-f341ebe06521\", \"content-length\": \"29\", \"date\": \"Wed, 24 Jun 2026 04:43:53 GMT\"} })"} {"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:43:53.271363303Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":2,"n_pruned_tasks":0} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.356401393Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.356437624Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:53.358689966Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:54.355881185Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:54.355948126Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:54.358677219Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.122847127Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.122893978Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.122911379Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.127193498Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.132067531Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.133276539Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.134446236Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.135581972Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:43:55.223796096Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.355854957Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.355904358Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:55.358365905Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:56.356578407Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:56.356641958Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:56.359385032Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:57.35658507Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:57.356635451Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:57.358956615Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:58.356144592Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:58.356204704Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:58.358669771Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:59.355853198Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:59.35590075Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:43:59.358324766Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.123553948Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.123595029Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.123627029Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.128007421Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.133553719Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.134767948Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.135934975Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.13703792Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:00.224433365Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.356594618Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.356635479Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:00.359104036Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:01.356483949Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:01.35653448Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:01.359225112Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:02.356108963Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:02.356161204Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:02.358722493Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:03.355883951Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:03.355942953Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:03.358663486Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:04.35616011Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:04.356244812Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:04.35915427Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.123325818Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.123378779Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.123394469Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.127502675Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.132337557Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.133662918Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.134935087Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.136155865Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:05.224352989Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.356402969Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.35645613Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:05.358766234Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:06.356022493Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:06.356083985Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:06.358365178Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:07.35661269Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:07.356699092Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:07.359104057Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:44:08.199275227Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.199342678Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.199523452Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-06-24T04:44:08.199548133Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:603","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.356534791Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.356622493Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.356660304Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.356673114Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.358746532Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:08.360032802Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":36,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:09.356588215Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:09.356627686Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:09.359250873Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.123516807Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":166} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.123550657Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":166} {"msg":"unrolling 166 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.123557688Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.129084296Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.134505911Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_i16","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.135818482Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":170} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.137016024Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u16","n_rows":12} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.138192887Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u32","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.139405655Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u8","n_rows":12} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.140622013Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":88} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.141872642Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":136} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.143013128Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:10.224426885Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.356626499Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.35668326Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:10.359350602Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:11.356574161Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:11.356613011Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:11.359188381Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:12.356902522Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:12.356954873Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:12.359479101Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:13.356647409Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:13.35670552Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:13.359158087Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:14.356345204Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:14.356379725Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:14.358575426Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.123947972Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.123978143Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.123990653Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.12801644Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.132814668Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.134002995Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.135179983Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.136265088Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:15.224615045Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.356747957Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.356790338Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:15.359763337Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:16.356920994Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:16.356948295Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:16.359393951Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:17.356571249Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:17.35660224Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:17.358836422Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:18.357105175Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:18.357158986Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:18.359771807Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:19.356990905Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:19.357044467Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:19.359618806Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.124043951Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.124082652Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.124291296Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.128930774Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.133648103Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.134918343Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.136158901Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.137305858Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:20.224456378Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.356510868Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.356541708Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:20.358807211Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:21.35695711Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:21.356986891Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:21.359220043Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:22.356893176Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:22.356924311Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:22.359272827Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:44:23.200434469Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:23.200520471Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:23.200728406Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-06-24T04:44:23.200751226Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:603","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:23.356845453Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:23.356878244Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:23.359792202Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:24.356945569Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:24.35698329Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:24.359320294Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.124470264Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.124502935Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.124510705Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.128882616Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.134060826Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.135426888Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.136686397Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.137913276Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:25.226297354Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.357360281Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.357392692Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:25.359714445Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:26.357428436Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:26.357464037Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:26.360258702Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:27.357858629Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:27.35789654Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:27.361621726Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:28.356747296Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:28.356787856Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:28.359390377Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:29.357318072Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:29.357354293Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:29.360854014Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.124030179Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.12406639Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.12407471Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.128353899Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.133199502Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.134743338Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.136228422Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.13743827Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:30.227172289Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.357243964Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.357277314Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:30.360299724Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:31.357454072Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:31.357486052Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:31.359727904Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:32.357579588Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:32.357614829Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:32.360207119Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:33.357377247Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:33.357460748Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:33.360310344Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:34.357711977Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:34.357743168Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:34.3600105Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.124178428Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.124214729Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.124222849Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.128724183Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.13374338Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.135597693Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.137032066Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.138299756Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:35.225485396Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.357566146Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.357597407Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:35.360845813Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:36.357069628Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:36.357108559Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:36.359462563Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:37.357633064Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:37.357679355Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:37.360455289Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-24T04:44:38.200641519Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"} {"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.20068558Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending","method":"GET"} {"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.200917315Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-06-24T04:44:38.200941166Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:603","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/3644ccd5-3d06-4276-907c-a58b9b7d5591/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"} {"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357015293Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357037423Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357147246Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357182827Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357190527Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.357196707Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.359346707Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":35101,"producer_ip":"fd00:1122:3344:101::2","producer_id":"a82f25bb-1de2-45a8-b222-bbc959ce80ca","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":36,"n_results":1} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:38.359576912Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:39.357584679Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:39.35762894Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:39.360150218Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.124348817Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":168} {"msg":"unrolling 168 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.124382927Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.124390547Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":168} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.129903815Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.135779111Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_i16","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.137565383Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.13919189Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":172} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.140510671Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u16","n_rows":16} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.143204574Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u32","n_rows":24} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.144414111Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_u8","n_rows":12} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.14565409Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":92} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.146899009Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":138} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.148020815Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:40.225207944Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.357285635Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.357322815Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:40.359984377Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:41.357193876Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:41.357246337Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:41.360195575Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:42.357204689Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:42.35724692Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:42.360459475Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:43.357668613Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:43.357737757Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:43.361314617Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:44.357703786Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:44.357783428Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:44.36132053Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.124464905Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.124501526Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.124510356Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.128709353Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.133537155Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.134754013Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.13593764Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.137034086Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:45.225636999Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.357688429Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.35772625Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:45.360014953Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:46.357196411Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:46.357237872Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:46.359770681Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:47.357950872Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:47.357986653Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:47.360348718Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:48.357533255Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:48.357576787Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:48.361055427Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:49.357556369Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:49.35759936Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:49.362636527Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.124812139Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.12484351Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":130} {"msg":"unrolling 130 total samples","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.12485074Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.129200571Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.134546195Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_string","n_rows":50} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.135879666Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"fields_uuid","n_rows":52} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.137186626Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100} {"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.138416915Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"id":"af193533-7031-41da-aa01-6ea0cef51994","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30} {"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-06-24T04:44:50.225636376Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:269","error":"Failed to check out connection to database: No backends found for this service"} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.357688026Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.357718946Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:50.360193444Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:51.357660618Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:51.35775328Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:51.360570996Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1} {"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:52.357482287Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:52.357521958Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent"} {"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-06-24T04:44:52.359912804Z","hostname":"oxz_oximeter_3644ccd5-3d06-4276-907c-a58b9b7d5591","pid":25155,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"291556d0-fe2a-4212-b80c-cfbc21f5c160","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"3644ccd5-3d06-4276-907c-a58b9b7d5591","component":"oximeter-agent","n_samples":26,"n_results":1}