[ Sep  4 23:14:29 Enabled. ]
[ Sep  4 23:14:29 Rereading configuration. ]
[ Sep  4 23:14:30 Rereading configuration. ]
[ Sep  4 23:14:32 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 5baecb58-7acf-464f-a54c-b20f17e8629d &"). ]
[ Sep  4 23:14:32 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:33.061644941Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:33.062810999Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:299"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:33.063019954Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:33.071233477Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:39.365289372Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.365333693Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:39.365342593Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:788"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:39.366862279Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:39.36689052Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:793","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:39.36689912Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:798"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.36690856Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:39.461968103Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:819","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.47670026Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"4b24fdf6-5d5d-4a60-be2a-a0e5948ff1be","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.480938Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.480958911Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.480972411Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.480980021Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:39.480988551Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.480997111Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.482420955Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.482439625Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.495493524Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.496850025Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 170.086949ms","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:39.496872956Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.667988293Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.668091355Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:39.668236088Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 423.250443ms","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:39.668248689Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.092331952Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.092403924Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.092561318Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 591.147094ms","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:40.092598549Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.683846076Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.683938038Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:40.684185804Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.957552088s","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:40.684217904Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:413","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:43.64313697Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:43.643209342Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:43.668538569Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"56bdf898-b563-4bde-a7ae-841e493df112\", \"date\": \"Fri, 04 Sep 2026 23:14:43 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:43.668672052Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"file":"oximeter/collector/src/lib.rs:431","id":"5baecb58-7acf-464f-a54c-b20f17e8629d"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:43.668685273Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:626"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:43.669609705Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:43.669712897Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:43.699664353Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ddd4aef-4743-44b0-baee-260f540c5e00\", \"content-length\": \"29\", \"date\": \"Fri, 04 Sep 2026 23:14:43 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:43.699734955Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.476885937Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":4}
{"msg":"inserting samples into database","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.476956368Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":4}
{"msg":"unrolling 4 total samples","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.476968609Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"fetching timeseries schema from database","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.47700747Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.478948815Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_queue_depth\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: HistogramU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-09-04T23:14:44.476987909Z }","timeseries_name":"oximeter_collector:database_queue_depth"}
{"msg":"new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.479028517Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","schema":"TimeseriesSchema { timeseries_name: TimeseriesName(\"oximeter_collector:database_samples_dropped\"), description: TimeseriesDescription { target: \"\", metric: \"\" }, field_schema: {FieldSchema { name: \"collector_id\", field_type: Uuid, source: Target, description: \"\" }, FieldSchema { name: \"collector_ip\", field_type: IpAddr, source: Target, description: \"\" }, FieldSchema { name: \"collector_port\", field_type: U16, source: Target, description: \"\" }, FieldSchema { name: \"collector_sink\", field_type: String, source: Metric, description: \"\" }}, datum_type: CumulativeU64, version: 1, authz_scope: Fleet, units: Count, created: 2026-09-04T23:14:44.476984319Z }","timeseries_name":"oximeter_collector:database_samples_dropped"}
{"msg":"inserting 2 new timeseries schema","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.479237512Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.48550027Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"fields_ipaddr","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.488598673Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"fields_string","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.491595104Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"fields_u16","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.496249373Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"fields_uuid","n_rows":4}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.498808214Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"measurements_cumulativeu64","n_rows":2}
{"msg":"inserted rows into table","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:44.502807988Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"id":"7379895a-91c1-403b-a454-fa03079ce7b7","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","table_name":"measurements_histogramu64","n_rows":2}
{"msg":"failed to insert some results into metric DB","v":0,"name":"oximeter","level":40,"time":"2026-09-04T23:14:44.578052833Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/results_sink.rs:276","error":"Failed to check out connection to database: No backends found for this service"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:49.477294162Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:49.477367734Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:54.476651695Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:54.476693966Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:58.669113023Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:629"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:58.669157284Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:58.680198414Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f849679e-3ffa-4538-b13f-bd8da0a38dda\", \"content-length\": \"298\", \"date\": \"Fri, 04 Sep 2026 23:14:58 GMT\"} })"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:58.680239645Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJiM2M5NjE4Mi03NzYyLTQ2ZTctYWI0My0zY2M1N2I4OWRiZWMifX0%3D","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:58.687937167Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/5baecb58-7acf-464f-a54c-b20f17e8629d/producers?page_token=eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJiM2M5NjE4Mi03NzYyLTQ2ZTctYWI0My0zY2M1N2I4OWRiZWMifX0%3D\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4df90305-d63a-4ab8-9dcb-4519276ee09d\", \"content-length\": \"29\", \"date\": \"Fri, 04 Sep 2026 23:14:58 GMT\"} })"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:58.688001939Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:8001","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-09-04T23:14:58.688022149Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:704","n_current_tasks":1,"n_pruned_tasks":0}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:59.477131213Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:59.477167044Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:59.501168686Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:59.501199296Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:14:59.523838953Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:00.501047323Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:00.501085793Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:00.502989278Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:01.501168303Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:01.501209004Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:01.502683309Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":26,"n_results":1}
{"msg":"sent timer-based collection request to the collection task","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:02.500889734Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:02.500928095Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2026-09-04T23:15:02.502612485Z","hostname":"oxz_oximeter_5baecb58-7acf-464f-a54c-b20f17e8629d","pid":22496,"producer_port":8001,"producer_ip":"fd00:1122:3344:101::1","producer_id":"b3c96182-7762-46e7-ab43-3cc57b89dbec","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"5baecb58-7acf-464f-a54c-b20f17e8629d","component":"oximeter-agent","n_samples":26,"n_results":1}