[ Dec 13 05:14:33 Disabled. ]
[ Dec 13 05:14:33 Rereading configuration. ]
[ Dec 13 05:14:43 Rereading configuration. ]
[ Dec 13 05:14:43 Enabled. ]
[ Dec 13 05:14:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 916c7ed2-6f94-4b70-b159-8507ba9f8d1b &"). ]
[ Dec 13 05:14:44 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.247230166Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:44.253124147Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"file":"oximeter/collector/src/lib.rs:192"}
{"msg":"new DNS resolver","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:44.264758586Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.271621651Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.273612822Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:44.388243577Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"2933772a-5c2d-4c52-85d7-79b3d4ae2430","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:44.392711606Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"283.917698ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.677806285Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.677839905Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:44.709447172Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"c7a05970-88bd-4acd-bf04-ba7c8ba9a83d","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:44.709916359Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"275.991059ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.987004005Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:44.987039236Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:45.019382942Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"62b74ffc-172b-4c19-b318-cdffe0e446d0","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:868","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:45.019807655Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"file":"oximeter/collector/src/lib.rs:213","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)\"))","retry_after":"1.184343089s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.209100073Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"lookup srv","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.209162915Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","dns_name":"_clickhouse._tcp.control-plane.oxide.internal"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:46.272144437Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.272211554Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:46.279877996Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:802"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2023-12-13T05:14:46.28113141Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:864"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:46.28117462Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:806","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:46.281238296Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:811"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.281245834Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.285635851Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:46.454004234Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"id":"a66f1274-1730-4fac-956f-49989906693a","component":"clickhouse-client","collector_id":"916c7ed2-6f94-4b70-b159-8507ba9f8d1b","component":"oximeter-agent","file":"oximeter/db/src/client.rs:834","version":4}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.476517849Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.476541839Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.476578065Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.476586109Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2023-12-13T05:14:46.476630116Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:195"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.476667401Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.509859645Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205}
{"msg":"lookup_ipv6 srv","v":0,"name":"oximeter","level":20,"time":"2023-12-13T05:14:46.509875058Z","hostname":"oxz_oximeter_916c7ed2-6f94-4b70-b159-8507ba9f8d1b","pid":22205,"component":"DnsResolver","dns_name":"_nexus._tcp.control-plane.oxide.internal"}