[ Oct  4 22:29:11 Disabled. ]
[ Oct  4 22:29:11 Rereading configuration. ]
[ Oct  4 22:29:27 Rereading configuration. ]
[ Oct  4 22:29:27 Rereading configuration. ]
[ Oct  4 22:29:27 Enabled. ]
[ Oct  4 22:29:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 7540d58c-49bc-4ae5-8ded-5a15b67c1541 &"). ]
[ Oct  4 22:29:27 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:27.813295981Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:27.814596991Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:488"}
{"msg":"new DNS resolver","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:27.815975392Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:27.81970402Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:27.820322629Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:27.899870499Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"id":"46feb953-17a9-4541-9234-5e80f34b18ab","component":"clickhouse-client","collector_id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:28.040614465Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:28.040653465Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/fa728d0/dropshot/src/server.rs:195"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:28.040672076Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:28.073199038Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:28.073217038Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 170.022485ms","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:35.955746305Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:36.126378912Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:36.126423883Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 394.392073ms","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:36.127150824Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:36.5221709Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:36.52220234Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 1.202914725s","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:36.522885311Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:37.723769081Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:37.723831013Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 2.889256173s","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:37.724593924Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:40.606779652Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:40.606817593Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 3.086859785s","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:40.607454643Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:43.687058026Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:43.687120537Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 8.069804443s","v":0,"name":"oximeter","level":40,"time":"2023-10-04T22:29:43.688525578Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:560","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:51.737369404Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:51.737416325Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:51.789906226Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"file":"oximeter/collector/src/lib.rs:574","id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541"}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:51.965634102Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/fa728d0/dropshot/src/server.rs:769","remote_addr":"[fd00:1122:3344:101::a]:49508"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:51.966978063Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"collector_id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541","component":"oximeter-agent","file":"oximeter/collector/src/lib.rs:358","address":"[fd00:1122:3344:101::2]:12224","producer_id":"3b0bc631-fc13-4a90-85ee-070e83331555"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:51.967041564Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"uri":"/producers","method":"POST","req_id":"3f806caa-123a-4a62-b92a-14a2bf0f37fe","remote_addr":"[fd00:1122:3344:101::a]:49508","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/fa728d0/dropshot/src/server.rs:853","latency_us":1251,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:52.000279148Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"producer_id":"3b0bc631-fc13-4a90-85ee-070e83331555","component":"collection-task","collector_id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541","component":"oximeter-agent","interval":"10s"}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:52.637525488Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/fa728d0/dropshot/src/server.rs:769","remote_addr":"[fd00:1122:3344:101::a]:40447"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:52.63762812Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"collector_id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541","component":"oximeter-agent","file":"oximeter/collector/src/lib.rs:358","address":"[fd00:1122:3344:101::a]:12221","producer_id":"3749f4e1-a360-46d1-b5e9-d0266154f837"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2023-10-04T22:29:52.637695901Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"uri":"/producers","method":"POST","req_id":"0bce8fae-014e-455d-965e-26554494f9b1","remote_addr":"[fd00:1122:3344:101::a]:40447","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/fa728d0/dropshot/src/server.rs:853","latency_us":113,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2023-10-04T22:29:52.671101757Z","hostname":"oxz_oximeter_7540d58c-49bc-4ae5-8ded-5a15b67c1541","pid":22452,"producer_id":"3749f4e1-a360-46d1-b5e9-d0266154f837","component":"collection-task","collector_id":"7540d58c-49bc-4ae5-8ded-5a15b67c1541","component":"oximeter-agent","interval":"10s"}