[ Jul  1 16:44:28 Enabled. ]
[ Jul  1 16:44:28 Rereading configuration. ]
[ Jul  1 16:44:31 Rereading configuration. ]
[ Jul  1 16:44:38 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 2b0ee6e5-9404-42f2-8cc7-a026579489d3 &"). ]
[ Jul  1 16:44:38 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:38.898769935Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:38.899341355Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"file":"oximeter/collector/src/lib.rs:301"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:38.899903375Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:38.911524553Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-01T16:44:46.369273843Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.369381665Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:46.369439126Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-01T16:44:46.370478503Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:46.370584665Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:46.370625076Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.370667757Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:46.950980701Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"63d92135-bbfc-4cd0-8022-b70d7201cd86","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.968087833Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"id":"a2dc4d78-cc4b-4b4f-9a9c-4254927ba0d6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.97435995Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.97438832Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.974461901Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.974482292Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:46.974524142Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.978096683Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.978195475Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.978250436Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.996913874Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:46.997651027Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 282.944283ms","v":0,"name":"oximeter","level":40,"time":"2026-07-01T16:44:46.997712497Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.281003623Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.281099915Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.281340959Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 407.574239ms","v":0,"name":"oximeter","level":40,"time":"2026-07-01T16:44:47.28138419Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.690467817Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.690597759Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:47.690781273Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.350669124s","v":0,"name":"oximeter","level":40,"time":"2026-07-01T16:44:47.690831813Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:49.042440197Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:49.042510018Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:49.120267562Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"ba0a69c2-5beb-4782-9754-941f8445651b\", \"date\": \"Wed, 01 Jul 2026 16:44:49 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:49.121017325Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"file":"oximeter/collector/src/lib.rs:464","id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:49.121110717Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:562"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:49.122355718Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:49.124433743Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/2b0ee6e5-9404-42f2-8cc7-a026579489d3/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:49.275086129Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/2b0ee6e5-9404-42f2-8cc7-a026579489d3/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5ef9930-8091-4d14-a96c-812f28cad2ff\", \"content-length\": \"29\", \"date\": \"Wed, 01 Jul 2026 16:44:49 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-01T16:44:49.277706524Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:640","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:51.968764622Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:51.968840733Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:56.96889889Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"component":"database-inserter","component":"results-sink-cluster","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}
{"msg":"batch interval expired, but no samples to insert","v":0,"name":"oximeter","level":20,"time":"2026-07-01T16:44:56.968987931Z","hostname":"oxz_oximeter_2b0ee6e5-9404-42f2-8cc7-a026579489d3","pid":23420,"component":"database-inserter","component":"results-sink","collector_ip":"fd00:1122:3344:101::d","collector_id":"2b0ee6e5-9404-42f2-8cc7-a026579489d3","component":"oximeter-agent"}