[ Jun 11 16:44:49 Enabled. ]
[ Jun 11 16:44:49 Rereading configuration. ]
[ Jun 11 16:44:52 Rereading configuration. ]
[ Jun 11 16:44:54 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 57a6e8d5-5685-49da-a6d4-97936d8bbcd7 &"). ]
[ Jun 11 16:44:54 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:54.961087118Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:54.961684664Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"file":"oximeter/collector/src/lib.rs:301"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:54.961743176Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:54.979623072Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-11T16:44:59.485245531Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.485298492Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:59.485307873Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:790"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-06-11T16:44:59.48634668Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:890"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:59.486361681Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:795","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:59.486370241Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.486379181Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:59.580017996Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"688541b0-9af3-4417-a5b9-214e34e5c733","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.603216905Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"id":"0ee0c7f0-739c-46a3-b324-ed73d1a269a4","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"57a6e8d5-5685-49da-a6d4-97936d8bbcd7","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.609228805Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.609265096Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.609290177Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.609301707Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-06-11T16:44:59.609316587Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.0/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.611741992Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.611827374Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.611843945Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.629448024Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.630179553Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 248.740633ms","v":0,"name":"oximeter","level":40,"time":"2026-06-11T16:44:59.630237415Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.879416155Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.879570719Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:44:59.879766895Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 673.166139ms","v":0,"name":"oximeter","level":40,"time":"2026-06-11T16:44:59.879796565Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:45:00.553924581Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:45:00.554018413Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-06-11T16:45:00.554209588Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.037953139s","v":0,"name":"oximeter","level":40,"time":"2026-06-11T16:45:00.554233619Z","hostname":"oxz_oximeter_57a6e8d5-5685-49da-a6d4-97936d8bbcd7","pid":22849,"file":"oximeter/collector/src/lib.rs:446","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}