[ Nov 16 16:44:00 Disabled. ]
[ Nov 16 16:44:00 Rereading configuration. ]
[ Nov 16 16:44:21 Rereading configuration. ]
[ Nov 16 16:44:21 Rereading configuration. ]
[ Nov 16 16:44:21 Enabled. ]
[ Nov 16 16:44:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id e9a9fce0-a75d-4005-b97a-f68f1ca81cc7 &"). ]
[ Nov 16 16:44:21 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.495169153Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.497003433Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:192"}
{"msg":"new DNS resolver","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.498565025Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.502058635Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.502684523Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:21.611768628Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:658"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.611897872Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.619274154Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:596"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:21.620093102Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:658"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.620123477Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:600","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.620137296Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:605"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.620143999Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.620747461Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.783004836Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"id":"ace84110-47ef-4639-ba8f-bcbd427593c5","component":"clickhouse-client","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","file":"oximeter/db/src/client.rs:628","version":3}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.792198899Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.792216684Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.792227549Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.792234581Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:21.7922439Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:195"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.792250969Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.825292508Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:21.82531272Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 308.207987ms","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:21.99569197Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:22.305593309Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:22.305622653Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 401.044757ms","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:22.306199458Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:22.70939179Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:22.709431325Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 548.391348ms","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:22.710149463Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:23.260807633Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:23.260834699Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 1.758233625s","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:23.261430925Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:25.025370298Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:25.025403748Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 5.245989726s","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:25.026326108Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:30.287887965Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:30.287925813Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 8.970939078s","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:30.288705141Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:39.28519849Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:39.28524483Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"failed to contact nexus, will retry in 18.929331976s","v":0,"name":"oximeter","level":40,"time":"2023-11-16T16:44:39.286003733Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:271","error":"\"error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:58.264989338Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:58.265050418Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:58.330821614Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"file":"oximeter/collector/src/lib.rs:285","id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7"}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:59.222067042Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:769","remote_addr":"[fd00:1122:3344:101::b]:64937"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:59.226380017Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:12224","producer_id":"2f78c1b6-1cf5-4464-b2e1-32307ed893f9"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2023-11-16T16:44:59.226470568Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"uri":"/producers","method":"POST","req_id":"5b6c9a0e-8699-4f5f-9d9d-a3176b1e5791","remote_addr":"[fd00:1122:3344:101::b]:64937","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:853","latency_us":4248,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2023-11-16T16:44:59.260247371Z","hostname":"oxz_oximeter_e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","pid":22511,"producer_id":"2f78c1b6-1cf5-4464-b2e1-32307ed893f9","component":"collection-task","collector_id":"e9a9fce0-a75d-4005-b97a-f68f1ca81cc7","component":"oximeter-agent","interval":"10s"}