[ Feb 12 21:14:36 Enabled. ]
[ Feb 12 21:14:36 Rereading configuration. ]
[ Feb 12 21:14:39 Rereading configuration. ]
[ Feb 12 21:14:45 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 0b0f000e-5f70-455c-b681-1e398c01c82c &"). ]
[ Feb 12 21:14:45 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:45.394349338Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:45.40090772Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"file":"oximeter/collector/src/lib.rs:192"}
{"msg":"new DNS resolver","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:45.404187057Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:45.407423552Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:45.409551049Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:45.4962225Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"e16ac5a9-4957-43bf-8520-42423580df4c","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:45.496845565Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"365.12518ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:45.862971813Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:45.863012533Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:45.893524137Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"5ce9d9c7-f597-40a1-bab5-96f066aa5ff8","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:45.8939725Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"749.171788ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:46.644106739Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:46.644145669Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:46.674021938Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"5f7d4d36-9695-452c-bd7e-efe7ed9b0323","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:46.674430531Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"745.791156ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.421531257Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.421570197Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:47.47720866Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.47724177Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:47.484094185Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:802"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-02-12T21:14:47.484886911Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:47.484902682Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:806","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:47.484923842Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:811"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.484930632Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.487445752Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:47.653219123Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"id":"0ced844a-8822-4695-b8e1-b3cb81feff78","component":"clickhouse-client","collector_id":"0b0f000e-5f70-455c-b681-1e398c01c82c","component":"oximeter-agent","file":"oximeter/db/src/client.rs:834","version":4}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.67668258Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.67670198Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.67671408Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.67672086Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2024-02-12T21:14:47.67673147Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.680210108Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.711551107Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2024-02-12T21:14:47.711566398Z","hostname":"oxz_oximeter_0b0f000e-5f70-455c-b681-1e398c01c82c","pid":24008,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}