[ Jan 19 22:44:05 Disabled. ]
[ Jan 19 22:44:05 Rereading configuration. ]
[ Jan 19 22:44:38 Rereading configuration. ]
[ Jan 19 22:44:38 Rereading configuration. ]
[ Jan 19 22:44:38 Enabled. ]
[ Jan 19 22:44:38 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id d7d109ec-98c1-4633-aba6-b6805122a2a5 &"). ]
[ Jan 19 22:44:38 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.003119922Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.005741471Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:192"}
{"msg":"new DNS resolver","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.007115647Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.01665958Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.016794634Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:39.120734425Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.120891059Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.128239234Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:802"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:39.129061315Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.129075136Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:806","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.129086626Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:811"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.129092946Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.129745284Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.512419773Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"id":"f744b5f0-4686-47d9-b8d6-3796b1b2e7fe","component":"clickhouse-client","collector_id":"d7d109ec-98c1-4633-aba6-b6805122a2a5","component":"oximeter-agent","file":"oximeter/db/src/client.rs:834","version":4}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.524118353Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.524139383Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.524152704Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.524159184Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:39.524167744Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/b19a9a5/dropshot/src/server.rs:195"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.524174074Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.555860103Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:39.555877424Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 312.760283ms","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:45.933682176Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:46.247768Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:46.247799271Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 381.054608ms","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:46.248430118Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:46.630504181Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:46.630547043Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 832.196721ms","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:46.631129748Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:47.46422314Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:47.464257601Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 1.385590915s","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:47.466046848Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:48.853155246Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:48.853182987Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 3.786441322s","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:48.853852914Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:52.641688739Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:52.641740261Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 6.494654618s","v":0,"name":"oximeter","level":40,"time":"2024-01-19T22:44:52.643071286Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:59.139343135Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-01-19T22:44:59.139401986Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2024-01-19T22:44:59.186152304Z","hostname":"oxz_oximeter_d7d109ec-98c1-4633-aba6-b6805122a2a5","pid":24224,"file":"oximeter/collector/src/lib.rs:285","id":"d7d109ec-98c1-4633-aba6-b6805122a2a5"}