[ Jun 26 12:44:40 Enabled. ]
[ Jun 26 12:44:40 Rereading configuration. ]
[ Jun 26 12:44:43 Rereading configuration. ]
[ Jun 26 12:44:49 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 15e00048-4ca6-45db-ae22-e8e5223c83f4 &"). ]
[ Jun 26 12:44:49 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:49.504521387Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:49.506088929Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:301"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:49.506136291Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:49.518638164Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:51.959950355Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:51.960036318Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:51.96007982Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:51.961158945Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:51.961229547Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:51.961239228Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:51.961278769Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:52.141689151Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"6b253ddd-4e51-43f7-973f-995d993c4bb5","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.160968978Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"id":"cafb94f0-c621-47f0-ba38-3b07a568fb9d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.168691463Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.168729515Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.168787996Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.168796937Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:52.168842378Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.0/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.172383325Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.172465368Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.172508609Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.192014534Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.192660135Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 314.116482ms","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:52.192727288Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.507908713Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.508005086Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:52.508221693Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 660.656783ms","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:52.508322126Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.169371681Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.169475224Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.169691741Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 576.034353ms","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:53.169708461Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.746793872Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.746894105Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:53.747099792Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.022592353s","v":0,"name":"oximeter","level":40,"time":"2026-06-26T12:44:53.747170634Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:55.770231597Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:55.77032807Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:55.796737443Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"811aa99d-ef91-4f9b-a91e-4e33b89879b4\", \"date\": \"Fri, 26 Jun 2026 12:44:55 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:55.796826075Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"file":"oximeter/collector/src/lib.rs:464","id":"15e00048-4ca6-45db-ae22-e8e5223c83f4"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:55.796889608Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:562"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:55.797936682Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:55.800520768Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/15e00048-4ca6-45db-ae22-e8e5223c83f4/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:55.846803937Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/15e00048-4ca6-45db-ae22-e8e5223c83f4/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a9ec65e-cceb-4df1-b656-a5bad0582f85\", \"content-length\": \"29\", \"date\": \"Fri, 26 Jun 2026 12:44:55 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-06-26T12:44:55.846917321Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:57.162878867Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-06-26T12:44:57.16298186Z","hostname":"oxz_oximeter_15e00048-4ca6-45db-ae22-e8e5223c83f4","pid":24182,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"15e00048-4ca6-45db-ae22-e8e5223c83f4","component":"oximeter-agent"}