[ Jul  2 21:44:46 Enabled. ]
[ Jul  2 21:44:47 Rereading configuration. ]
[ Jul  2 21:44:48 Rereading configuration. ]
[ Jul  2 21:44:51 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 84093dc6-9d7e-44c3-ac89-48bb7e36e692 &"). ]
[ Jul  2 21:44:51 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:52.125849447Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:52.148682491Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"file":"oximeter/collector/src/lib.rs:301"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:52.150351431Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:52.157044905Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-02T21:44:57.032517918Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.032565128Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:57.032575748Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-02T21:44:57.033554231Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:57.033602781Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:57.033612631Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.033622572Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:57.122334326Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"e031e518-cf34-4f15-856c-1ba1c1f3946d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.140203199Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"id":"5a3ae46d-7895-47b9-92aa-824b95a4672e","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.155744552Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.155781923Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.155820053Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.155829494Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:57.155841403Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.15876362Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.158832551Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.158847891Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.176579672Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.177185259Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 282.150886ms","v":0,"name":"oximeter","level":40,"time":"2026-07-02T21:44:57.17721377Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.460333756Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.460461797Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:57.460709291Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 729.777587ms","v":0,"name":"oximeter","level":40,"time":"2026-07-02T21:44:57.460763341Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:58.191798276Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:58.191910467Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:58.303088162Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"aea6aa7b-9c86-435a-a8b4-daa33495bb3b\", \"date\": \"Thu, 02 Jul 2026 21:44:58 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:58.3077552Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"file":"oximeter/collector/src/lib.rs:464","id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:58.307856601Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:562"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-07-02T21:44:58.308904454Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:565"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:58.316511439Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/84093dc6-9d7e-44c3-ac89-48bb7e36e692/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-02T21:44:58.316801242Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/84093dc6-9d7e-44c3-ac89-48bb7e36e692/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2026-07-02T21:44:58.3174162Z","hostname":"oxz_oximeter_84093dc6-9d7e-44c3-ac89-48bb7e36e692","pid":22443,"collector_ip":"fd00:1122:3344:101::d","collector_id":"84093dc6-9d7e-44c3-ac89-48bb7e36e692","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:603","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/84093dc6-9d7e-44c3-ac89-48bb7e36e692/producers?sort_by=id_ascending): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors/84093dc6-9d7e-44c3-ac89-48bb7e36e692/producers?sort_by=id_ascending): client error (Connect): tcp connect error: Connection refused (os error 146)"}