[ Jan 30 03:44:34 Enabled. ]
[ Jan 30 03:44:34 Rereading configuration. ]
[ Jan 30 03:44:36 Rereading configuration. ]
[ Jan 30 03:44:43 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 77864081-00ec-45aa-b22f-bdce0464539f &"). ]
[ Jan 30 03:44:43 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:43.516375593Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:43.518665632Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"file":"oximeter/collector/src/lib.rs:264"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:43.520638223Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:43.560340981Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-01-30T03:44:48.027612282Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:48.027701874Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:48.027750685Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-01-30T03:44:48.029524411Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:48.029587272Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:805","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:48.029597382Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:810"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:48.029638654Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:49.050464169Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"dcb97f77-4c0d-4b88-9010-01c4cbcef3f9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:831","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.106223958Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"id":"a325c45a-ae01-48c8-82cb-33ce26fb09eb","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.135972371Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.135999352Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.136055183Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.136065773Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:49.136111235Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.6/src/server.rs:328"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.163829785Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.163886317Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.163936958Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.420746984Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.436367424Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 226.239314ms","v":0,"name":"oximeter","level":40,"time":"2026-01-30T03:44:49.436430886Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.663546069Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.663638962Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:49.663866438Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 688.048211ms","v":0,"name":"oximeter","level":40,"time":"2026-01-30T03:44:49.663932269Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:50.353044938Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:50.353141601Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:50.701902583Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"5766c195-afc6-4f75-9257-e9eabc279bf2\", \"date\": \"Fri, 30 Jan 2026 03:44:50 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:50.702000656Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"file":"oximeter/collector/src/lib.rs:427","id":"77864081-00ec-45aa-b22f-bdce0464539f"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:50.702093598Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:554"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:50.703129345Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:557"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:50.705757882Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/77864081-00ec-45aa-b22f-bdce0464539f/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-01-30T03:44:50.959386845Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/77864081-00ec-45aa-b22f-bdce0464539f/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6289e629-9e19-47c6-b105-fc5aac0e9cd6\", \"content-length\": \"29\", \"date\": \"Fri, 30 Jan 2026 03:44:50 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-01-30T03:44:50.959465077Z","hostname":"oxz_oximeter_77864081-00ec-45aa-b22f-bdce0464539f","pid":24774,"collector_ip":"fd00:1122:3344:101::d","collector_id":"77864081-00ec-45aa-b22f-bdce0464539f","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:632","n_current_tasks":0,"n_pruned_tasks":0}