[ Jul  8 00:44:49 Enabled. ]
[ Jul  8 00:44:49 Rereading configuration. ]
[ Jul  8 00:44:50 Rereading configuration. ]
[ Jul  8 00:44:53 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 966848da-ebae-47e5-8ea7-292193a7ff3f &"). ]
[ Jul  8 00:44:53 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:53.34776906Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:53.349216992Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"file":"oximeter/collector/src/lib.rs:299"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:53.350070921Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:53.364211539Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:44:58.072686729Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.07272776Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:58.07273873Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:788"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:44:58.073686422Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:888"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:58.073725572Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:793","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:58.073735053Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:798"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.073743253Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:58.159317038Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"5f7c8e36-b3cd-46e1-8bc3-748222116b9b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:819","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.180418503Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"id":"4f63655a-f349-472a-a7b0-26477d428ad1","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"966848da-ebae-47e5-8ea7-292193a7ff3f","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.187563504Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.187582674Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.187595715Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.187602455Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-07-08T00:44:58.187612875Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.190581742Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.190652714Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.190700905Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.210809977Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.211688597Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 171.800281ms","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:44:58.211719908Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"file":"oximeter/collector/src/lib.rs:444","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.384824672Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.384948005Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.385213681Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 476.665395ms","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:44:58.385228771Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"file":"oximeter/collector/src/lib.rs:444","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.862384239Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.862488931Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:44:58.86287094Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.430393689s","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:44:58.862936241Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"file":"oximeter/collector/src/lib.rs:444","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:45:00.294964185Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:45:00.295077638Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-07-08T00:45:00.295259112Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.610777935s","v":0,"name":"oximeter","level":40,"time":"2026-07-08T00:45:00.295322943Z","hostname":"oxz_oximeter_966848da-ebae-47e5-8ea7-292193a7ff3f","pid":22795,"file":"oximeter/collector/src/lib.rs:444","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}