[ Mar 24 18:29:36 Enabled. ]
[ Mar 24 18:29:36 Rereading configuration. ]
[ Mar 24 18:29:40 Rereading configuration. ]
[ Mar 24 18:29:45 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 240d30ab-078d-431f-bd03-eb34f6d6408a &"). ]
[ Mar 24 18:29:45 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:46.169657231Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:46.175947224Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"file":"oximeter/collector/src/lib.rs:265"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:46.176004425Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:46.225330981Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-03-24T18:29:53.747632197Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:53.747703177Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:53.747736908Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-03-24T18:29:53.749710114Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:53.749762124Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:805","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:53.749770905Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:810"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:53.749807275Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:54.116522821Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"06c8da40-fc20-4942-ac23-88d498ec03e4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:831","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.157978534Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"id":"18b69501-b0f1-471f-b3ee-c6a35e5bc415","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.182572916Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.182606697Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.182673548Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.182683438Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:54.182752989Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.197603113Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.197728015Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.197790466Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.274860076Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.278320471Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 172.841889ms","v":0,"name":"oximeter","level":40,"time":"2026-03-24T18:29:54.278392612Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"file":"oximeter/collector/src/lib.rs:410","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.452105408Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.45221231Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.452415383Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 305.535203ms","v":0,"name":"oximeter","level":40,"time":"2026-03-24T18:29:54.452455783Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"file":"oximeter/collector/src/lib.rs:410","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.759554737Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:54.759674679Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:55.020316554Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"2df624e1-272b-4f01-a5b8-9d481c254dff\", \"date\": \"Tue, 24 Mar 2026 18:29:55 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:55.020393346Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"file":"oximeter/collector/src/lib.rs:428","id":"240d30ab-078d-431f-bd03-eb34f6d6408a"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:55.020465987Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:554"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-03-24T18:29:55.02152037Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:557"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:55.028115407Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/240d30ab-078d-431f-bd03-eb34f6d6408a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-03-24T18:29:55.028523092Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/240d30ab-078d-431f-bd03-eb34f6d6408a/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-03-24T18:29:55.030963144Z","hostname":"oxz_oximeter_240d30ab-078d-431f-bd03-eb34f6d6408a","pid":23782,"collector_ip":"fd00:1122:3344:101::d","collector_id":"240d30ab-078d-431f-bd03-eb34f6d6408a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:595","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/240d30ab-078d-431f-bd03-eb34f6d6408a/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/240d30ab-078d-431f-bd03-eb34f6d6408a/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"}