[ Aug 22 03:58:47 Enabled. ]
[ Aug 22 03:58:47 Rereading configuration. ]
[ Aug 22 03:58:51 Rereading configuration. ]
[ Aug 22 03:59:03 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 0f04d198-84a3-4816-9597-5d7e6bf0574a &"). ]
[ Aug 22 03:59:03 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:03.743545944Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:03.745160653Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"file":"oximeter/collector/src/lib.rs:299"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:03.745223044Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:03.77317629Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:08.208916235Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.209010066Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.209054547Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:788"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:08.210177728Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.210249869Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:793","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.210258809Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:798"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.21031181Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.525929238Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:819","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.553413656Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"234db6a6-3323-495d-88ff-66eb39869cc5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.565492955Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.565522186Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.565620427Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.565636838Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.565705179Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.573752945Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.573867047Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.573911598Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.610661704Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.786886037Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"2c46bf82-9214-42d6-ab8b-0f57d1ce512d\", \"date\": \"Sat, 22 Aug 2026 03:59:08 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.786953698Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"file":"oximeter/collector/src/lib.rs:431","id":"0f04d198-84a3-4816-9597-5d7e6bf0574a"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.786994379Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:626"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.78813714Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.79091323Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:08.903822246Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b10fef41-868e-463d-b89e-87f3021824e2\", \"content-length\": \"29\", \"date\": \"Sat, 22 Aug 2026 03:59:08 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:08.905979665Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.555170404Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":4}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.555221675Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":4}
{"msg":"unrolling 4 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.555326297Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.559007544Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.561185353Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_queue_depth\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:13.558954323Z }","timeseries_name":"oximeter_collector:database_queue_depth"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.561313486Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_samples_dropped\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:13.558942793Z }","timeseries_name":"oximeter_collector:database_samples_dropped"}
{"msg":"inserting 2 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.561498359Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.568871782Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.571663763Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.574726559Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.579819161Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.582924617Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:13.593066441Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":2}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:13.658898954Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:18.554908481Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:18.554993022Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.555404637Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.555446028Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:23.788086113Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.788163195Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.805248474Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4898d2b-e8db-4fa9-9984-4fd521f97c82\", \"content-length\": \"311\", \"date\": \"Sat, 22 Aug 2026 03:59:23 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.805323235Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIxNjY4NWQzNy1iNGNkLTRlNzQtOWFmYy1iMDZhNTA5MTE2NDUifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.814539622Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiIxNjY4NWQzNy1iNGNkLTRlNzQtOWFmYy1iMDZhNTA5MTE2NDUifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9a2d832-0472-4250-ade1-321646cf8da5\", \"content-length\": \"29\", \"date\": \"Sat, 22 Aug 2026 03:59:23 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:23.814606784Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:47533","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:23.814618104Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":1,"n_pruned_tasks":0}
{"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:27.600782405Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.554990894Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":2}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.555062226Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":2}
{"msg":"unrolling 2 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.555105766Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.555114267Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.561195427Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:collections\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"base_route\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"producer_id\", field_type: Uuid, source: Metric, description: \"\" }, FieldSchema { name: \"producer_ip\", field_type: IpAddr, source: Metric, description: \"\" }, FieldSchema { name: \"producer_port\", field_type: U16, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:28.555010475Z }","timeseries_name":"oximeter_collector:collections"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.561324149Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:samples_collected\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"base_route\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"producer_id\", field_type: Uuid, source: Metric, description: \"\" }, FieldSchema { name: \"producer_ip\", field_type: IpAddr, source: Metric, description: \"\" }, FieldSchema { name: \"producer_port\", field_type: U16, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:28.555014535Z }","timeseries_name":"oximeter_collector:samples_collected"}
{"msg":"inserting 2 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.561416541Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.565059513Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.566687436Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":2}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.568398267Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.570142499Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:28.571933961Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:28.656102036Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:32.817164132Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:32.817253714Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:32.847684065Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11,"n_results":2}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.55486442Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.554934281Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11}
{"msg":"unrolling 11 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.554977242Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.555023592Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.556487059Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"http_service:request_latency_histogram\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"name\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"operation_id\", field_type: String, source: Metric, description: \"\" }, FieldSchema { name: \"status_code\", field_type: U16, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:33.554905790Z }","timeseries_name":"http_service:request_latency_histogram"}
{"msg":"inserting 1 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.556707313Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.560787567Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":22}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.562377976Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":11}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.564005515Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":11}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:33.572663082Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":11}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:33.65584615Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.555407134Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.555449275Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:38.787691413Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.787765045Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.792189465Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26b9b385-bb70-453d-9a5f-a8098e6ffcb5\", \"content-length\": \"590\", \"date\": \"Sat, 22 Aug 2026 03:59:38 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.792266886Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJlNDhjODJkMS1hOWMyLTQxNjAtYWQyZS00MWI2MDExN2I5NzYifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.798094677Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJlNDhjODJkMS1hOWMyLTQxNjAtYWQyZS00MWI2MDExN2I5NzYifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"439872fb-f981-4dcb-9b71-6b45c0f1d247\", \"content-length\": \"29\", \"date\": \"Sat, 22 Aug 2026 03:59:38 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.798143334Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","address":"[fd00:1122:3344:101::c]:38996","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:38.798180479Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:34128","producer_id":"e48c82d1-a9c2-4160-ad2e-41b60117b976"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:38.798188466Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":3,"n_pruned_tasks":0}
{"msg":"reporting oximeter self-collection statistics","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:39.078326589Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":34128,"producer_ip":"fd00:1122:3344:101::1","producer_id":"e48c82d1-a9c2-4160-ad2e-41b60117b976","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:42.81756223Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:42.817637151Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:42.819584347Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11,"n_results":2}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.554787667Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":13}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.554890209Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":13}
{"msg":"unrolling 13 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.554909559Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.55495224Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.619331077Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.625336715Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":24}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.628132666Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":15}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.630682212Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":15}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.634379049Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:43.638310861Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":11}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:43.65651214Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.028339144Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.028419765Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.038983702Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":41,"n_results":5}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.555443824Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":41}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.555528466Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":41}
{"msg":"unrolling 41 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.555536586Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.555576397Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.55795776Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:cpus_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:48.555501755Z }","timeseries_name":"collection_target:cpus_provisioned"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.558012471Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:ram_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:48.555503655Z }","timeseries_name":"collection_target:ram_provisioned"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.558053411Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"collection_target:virtual_disk_space_provisioned\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: I64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:48.555505355Z }","timeseries_name":"collection_target:virtual_disk_space_provisioned"}
{"msg":"inserting 3 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.558779335Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.562669505Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":40}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.564284284Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":20}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.565967465Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":32}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.568207895Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":20}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:48.571868202Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_i64","n_rows":21}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:48.656049567Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:52.817094944Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:52.81745317Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:52.818787974Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":47533,"producer_ip":"fd00:1122:3344:101::2","producer_id":"16685d37-b4cd-4e74-9afc-b06a50911645","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11,"n_results":2}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.555008894Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.555102126Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":11}
{"msg":"unrolling 11 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.555154687Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.555204428Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.559638388Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":22}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.561554143Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":11}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.563451517Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":11}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.565972073Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":11}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:53.656148066Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:53.788305751Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.788399683Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.792608059Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"749f49a4-f729-4d0e-b2f6-3153a72650c3\", \"content-length\": \"726\", \"date\": \"Sat, 22 Aug 2026 03:59:53 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.79267477Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJlNDhjODJkMS1hOWMyLTQxNjAtYWQyZS00MWI2MDExN2I5NzYifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.795813077Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/0f04d198-84a3-4816-9597-5d7e6bf0574a/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJlNDhjODJkMS1hOWMyLTQxNjAtYWQyZS00MWI2MDExN2I5NzYifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c09805b9-186d-4df5-936a-efd4f64e0e05\", \"content-length\": \"29\", \"date\": \"Sat, 22 Aug 2026 03:59:53 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:53.795868038Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:8001","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-08-22T03:59:53.795915489Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":4,"n_pruned_tasks":0}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:54.058974416Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:54.059041337Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:54.064429695Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:55.059683208Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:55.060038994Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:55.061583462Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:56.05880842Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:56.058976173Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:56.060831257Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:57.059298364Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:57.059440904Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:57.061016719Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.028269384Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.028342626Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.030490365Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":38996,"producer_ip":"fd00:1122:3344:101::c","producer_id":"3889b162-1ea5-494e-aef3-4a60896b5f8e","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":23,"n_results":5}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.059655013Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.059762525Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.061570688Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.555773292Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":153}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.555808742Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":153}
{"msg":"unrolling 153 total samples","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.555853453Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.555895444Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558564202Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555795092Z }","timeseries_name":"ddm_router:originated_tunnel_endpoints"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558654744Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_router:originated_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555791642Z }","timeseries_name":"ddm_router:originated_underlay_prefixes"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558715915Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555804892Z }","timeseries_name":"ddm_session:advertisements_received"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558808477Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:advertisements_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555802582Z }","timeseries_name":"ddm_session:advertisements_sent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558856848Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_tunnel_endpoints\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555825083Z }","timeseries_name":"ddm_session:imported_tunnel_endpoints"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558905798Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:imported_underlay_prefixes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: U64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555823533Z }","timeseries_name":"ddm_session:imported_underlay_prefixes"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.558954939Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_address_changes\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555813522Z }","timeseries_name":"ddm_session:peer_address_changes"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.55900841Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_expirations\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555807102Z }","timeseries_name":"ddm_session:peer_expirations"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559060621Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:peer_sessions_established\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555815302Z }","timeseries_name":"ddm_session:peer_sessions_established"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559140573Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555800082Z }","timeseries_name":"ddm_session:solicitations_received"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559224584Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:solicitations_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555797692Z }","timeseries_name":"ddm_session:solicitations_sent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559271975Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:update_send_fail\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555820153Z }","timeseries_name":"ddm_session:update_send_fail"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559318186Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_received\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555818693Z }","timeseries_name":"ddm_session:updates_received"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.559364237Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"ddm_session:updates_sent\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"hostname\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"interface\", field_type: String, source: Target, description: \"\" }, FieldSchema { name: \"rack_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"sled_id\", field_type: Uuid, source: Target, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-08-22T03:59:58.555817102Z }","timeseries_name":"ddm_session:updates_sent"}
{"msg":"inserting 14 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.55953842Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.563568543Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_string","n_rows":96}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.565296574Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_u16","n_rows":23}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.566959724Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"fields_uuid","n_rows":75}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.568572534Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":100}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.570881346Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":23}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:58.572673308Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"id":"fa78fdc1-5865-4807-9761-8292811ec8f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","table_name":"measurements_u64","n_rows":30}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-08-22T03:59:58.656858643Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:59.058961979Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:59.059073881Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T03:59:59.060834333Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:00.058998829Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:00.059093281Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:00.060544737Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:01.059707862Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:01.059743402Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-08-22T04:00:01.061238289Z","hostname":"oxz_oximeter_0f04d198-84a3-4816-9597-5d7e6bf0574a","pid":26023,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"d6572c98-cc82-48e2-9ae5-fa115365134c","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"0f04d198-84a3-4816-9597-5d7e6bf0574a","component":"oximeter-agent","n_samples":26,"n_results":1}