[ Jan  5 21:59:34 Enabled. ]
[ Jan  5 21:59:35 Rereading configuration. ]
[ Jan  5 21:59:36 Rereading configuration. ]
[ Jan  5 21:59:41 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id b4305c20-1142-4888-9c13-36ecb3bb7ba2 &"). ]
[ Jan  5 21:59:41 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:41.576985226Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:41.579510722Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:264"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:41.587863786Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:41.629594744Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:45.241923458Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.241975419Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:45.241985019Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:45.243200256Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:45.243220697Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:805","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:45.243228817Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:810"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.243237117Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:45.394966506Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"51a35296-2bb1-4be8-be78-b7900761cafe","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:831","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.424138698Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"id":"bef5f708-4563-4386-8faa-25acb88e3b2d","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.446506111Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.446526541Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.446539391Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.446546992Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:45.446556042Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.6/src/server.rs:328"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.458648288Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.458696269Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.458709999Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.593303841Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.600453108Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 178.492856ms","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:45.600573761Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.779672513Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.779734994Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:45.779846027Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 310.226238ms","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:45.779858607Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:46.090954404Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:46.091078387Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:46.091297912Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.361857523s","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:46.091320712Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:47.453396961Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:47.453462603Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:47.453705218Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.495564963s","v":0,"name":"oximeter","level":40,"time":"2026-01-05T21:59:47.453744079Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:49.950716296Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:49.950821968Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:49.985110103Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"d72ee1ec-f4d3-4406-b88d-c8e8a3ea16b0\", \"date\": \"Mon, 05 Jan 2026 21:59:49 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:49.989711474Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"file":"oximeter/collector/src/lib.rs:427","id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:49.991688408Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:554"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:49.99271941Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:557"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:50.005354478Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/b4305c20-1142-4888-9c13-36ecb3bb7ba2/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-05T21:59:50.097364094Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/b4305c20-1142-4888-9c13-36ecb3bb7ba2/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6831e3e1-0bbd-4292-af56-b3adb04194d8\", \"content-length\": \"29\", \"date\": \"Mon, 05 Jan 2026 21:59:50 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-05T21:59:50.097436145Z","hostname":"oxz_oximeter_b4305c20-1142-4888-9c13-36ecb3bb7ba2","pid":23227,"collector_ip":"fd00:1122:3344:101::d","collector_id":"b4305c20-1142-4888-9c13-36ecb3bb7ba2","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:632","n_current_tasks":0,"n_pruned_tasks":0}