[ Apr 17 13:59:48 Enabled. ]
[ Apr 17 13:59:48 Rereading configuration. ]
[ Apr 17 13:59:50 Rereading configuration. ]
[ Apr 17 13:59:54 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id d843885d-fbc1-4b51-a1cd-6abe20785d4a &"). ]
[ Apr 17 13:59:54 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:54.638334538Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:54.641528575Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"file":"oximeter/collector/src/lib.rs:265"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:54.641610916Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:54.684655746Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-04-17T13:59:59.447161906Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.447205966Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:59.447215587Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-04-17T13:59:59.44844169Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:59.448456521Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:805","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:59.448462631Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:810"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.448469861Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:59.674339102Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"bd76b20c-40f2-4733-93f5-48099e13c7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:831","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.703492614Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"id":"9a0d2e4a-5d9b-4152-9e63-b38eef8304ea","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.720743501Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.720771981Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.720787101Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.720794401Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-04-17T13:59:59.720803691Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.728007733Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.728090965Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.728105505Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.772026215Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-04-17T13:59:59.772579441Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 353.007069ms","v":0,"name":"oximeter","level":40,"time":"2026-04-17T13:59:59.772597331Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"file":"oximeter/collector/src/lib.rs:410","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.126748514Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.126819285Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.127055448Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 287.822857ms","v":0,"name":"oximeter","level":40,"time":"2026-04-17T14:00:00.127095908Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"file":"oximeter/collector/src/lib.rs:410","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.41617063Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.416256821Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.651168785Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"9c27894e-10ff-4605-b305-d1396adcdef2\", \"date\": \"Fri, 17 Apr 2026 14:00:00 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-04-17T14:00:00.651247546Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"file":"oximeter/collector/src/lib.rs:428","id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-04-17T14:00:00.653431231Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:554"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-04-17T14:00:00.654495594Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:557"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.657201544Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"body":"None","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/d843885d-fbc1-4b51-a1cd-6abe20785d4a/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-04-17T14:00:00.756307533Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors/d843885d-fbc1-4b51-a1cd-6abe20785d4a/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f0f79ff-45ae-4ba7-acf0-919adafa99fc\", \"content-length\": \"29\", \"date\": \"Fri, 17 Apr 2026 14:00:00 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-04-17T14:00:00.756447764Z","hostname":"oxz_oximeter_d843885d-fbc1-4b51-a1cd-6abe20785d4a","pid":24173,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d843885d-fbc1-4b51-a1cd-6abe20785d4a","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:632","n_current_tasks":0,"n_pruned_tasks":0}