| 1 | | [ Feb 1 08:58:54 Enabled. ] |
| 2 | | [ Feb 1 08:58:54 Rereading configuration. ] |
| 3 | | [ Feb 1 08:58:56 Rereading configuration. ] |
| 4 | | [ Feb 1 08:59:00 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id e9ef6757-f42f-4ae5-8548-fb6ab94c30e9 &"). ] |
| 5 | | [ Feb 1 08:59:00 Method "start" exited with status 0. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-02-01T08:59:00.742Z | DEBG | oximeter: registered DTrace probes
|
| 8 | 2024-02-01T08:59:00.746Z | INFO | oximeter: starting oximeter server
file = oximeter/collector/src/lib.rs:192
|
| 9 | 2024-02-01T08:59:00.746Z | INFO | oximeter (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 10 | 2024-02-01T08:59:00.747Z | DEBG | oximeter: creating ClickHouse client
|
| 11 | 2024-02-01T08:59:00.747Z | DEBG | oximeter (DnsResolver): lookup srv
dns_name = _clickhouse._tcp.control-plane.oxide.internal
|
| 12 | 2024-02-01T08:59:00.815Z | WARN | oximeter (clickhouse-client): failed to read version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
error = Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)
file = oximeter/db/src/client.rs:868
id = e8a4d547-7eae-4bd0-8d22-fa527c8754e5
|
| 13 | 2024-02-01T08:59:00.816Z | WARN | oximeter: failed to create ClickHouse client
error = Database(DatabaseUnavailable("error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"))
file = oximeter/collector/src/lib.rs:213
retry_after = 294.426676ms
|
| 14 | 2024-02-01T08:59:01.113Z | DEBG | oximeter: creating ClickHouse client
|
| 15 | 2024-02-01T08:59:01.113Z | DEBG | oximeter (DnsResolver): lookup srv
dns_name = _clickhouse._tcp.control-plane.oxide.internal
|
| 16 | 2024-02-01T08:59:01.145Z | WARN | oximeter (clickhouse-client): failed to read version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
error = Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)
file = oximeter/db/src/client.rs:868
id = a2d28668-a288-4cc1-b361-674efc9136f8
|
| 17 | 2024-02-01T08:59:01.145Z | WARN | oximeter: failed to create ClickHouse client
error = Database(DatabaseUnavailable("error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"))
file = oximeter/collector/src/lib.rs:213
retry_after = 290.758902ms
|
| 18 | 2024-02-01T08:59:01.438Z | DEBG | oximeter: creating ClickHouse client
|
| 19 | 2024-02-01T08:59:01.439Z | DEBG | oximeter (DnsResolver): lookup srv
dns_name = _clickhouse._tcp.control-plane.oxide.internal
|
| 20 | 2024-02-01T08:59:01.471Z | WARN | oximeter (clickhouse-client): failed to read version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
error = Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)
file = oximeter/db/src/client.rs:868
id = 554da58e-12b1-4e9d-8beb-feb8b4f0d4a5
|
| 21 | 2024-02-01T08:59:01.471Z | WARN | oximeter: failed to create ClickHouse client
error = Database(DatabaseUnavailable("error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0): error trying to connect: tcp connect error: Connection refused (os error 146)"))
file = oximeter/collector/src/lib.rs:213
retry_after = 1.275790885s
|
| 22 | 2024-02-01T08:59:02.752Z | DEBG | oximeter: creating ClickHouse client
|
| 23 | 2024-02-01T08:59:02.752Z | DEBG | oximeter (DnsResolver): lookup srv
dns_name = _clickhouse._tcp.control-plane.oxide.internal
|
| 24 | 2024-02-01T08:59:02.801Z | WARN | oximeter (clickhouse-client): oximeter database does not exist, or is out-of-date
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:864
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 25 | 2024-02-01T08:59:02.801Z | DEBG | oximeter (oximeter-agent): oximeter database does not exist, creating
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
|
| 26 | 2024-02-01T08:59:02.806Z | INFO | oximeter (clickhouse-client): reading db version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:802
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 27 | 2024-02-01T08:59:02.806Z | WARN | oximeter (clickhouse-client): oximeter database does not exist, or is out-of-date
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:864
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 28 | 2024-02-01T08:59:02.806Z | INFO | oximeter (clickhouse-client): read oximeter database version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:806
id = d32c82d3-8a43-414e-920e-9f46d93a0222
version = 0
|
| 29 | 2024-02-01T08:59:02.806Z | INFO | oximeter (clickhouse-client): wiping and re-initializing oximeter schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:811
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 30 | 2024-02-01T08:59:02.806Z | DEBG | oximeter (clickhouse-client): wiping ClickHouse database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 31 | 2024-02-01T08:59:02.811Z | DEBG | oximeter (clickhouse-client): initializing ClickHouse database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 32 | 2024-02-01T08:59:03.272Z | INFO | oximeter (clickhouse-client): inserting current version
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
file = oximeter/db/src/client.rs:834
id = d32c82d3-8a43-414e-920e-9f46d93a0222
version = 4
|
| 33 | 2024-02-01T08:59:03.297Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = GET
path = /info
|
| 34 | 2024-02-01T08:59:03.297Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = GET
path = /producers
|
| 35 | 2024-02-01T08:59:03.297Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
path = /producers
|
| 36 | 2024-02-01T08:59:03.297Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = DELETE
path = /producers/{producer_id}
|
| 37 | 2024-02-01T08:59:03.297Z | INFO | oximeter (dropshot): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195
local_addr = [fd00:1122:3344:101::d]:12223
|
| 38 | 2024-02-01T08:59:03.300Z | DEBG | oximeter (dropshot): successfully registered DTrace USDT probes
local_addr = [fd00:1122:3344:101::d]:12223
|
| 39 | 2024-02-01T08:59:03.332Z | DEBG | oximeter: contacting nexus
|
| 40 | 2024-02-01T08:59:03.332Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 41 | 2024-02-01T08:59:07.444Z | WARN | oximeter: failed to contact nexus, will retry in 160.441579ms
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 42 | 2024-02-01T08:59:07.606Z | DEBG | oximeter: contacting nexus
|
| 43 | 2024-02-01T08:59:07.606Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 44 | 2024-02-01T08:59:07.607Z | WARN | oximeter: failed to contact nexus, will retry in 723.888167ms
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 45 | 2024-02-01T08:59:08.332Z | DEBG | oximeter: contacting nexus
|
| 46 | 2024-02-01T08:59:08.332Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 47 | 2024-02-01T08:59:08.333Z | WARN | oximeter: failed to contact nexus, will retry in 1.175306054s
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 48 | 2024-02-01T08:59:09.510Z | DEBG | oximeter: contacting nexus
|
| 49 | 2024-02-01T08:59:09.510Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 50 | 2024-02-01T08:59:09.511Z | WARN | oximeter: failed to contact nexus, will retry in 1.090750192s
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 51 | 2024-02-01T08:59:10.602Z | DEBG | oximeter: contacting nexus
|
| 52 | 2024-02-01T08:59:10.602Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 53 | 2024-02-01T08:59:10.603Z | WARN | oximeter: failed to contact nexus, will retry in 4.171581069s
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 54 | 2024-02-01T08:59:14.778Z | DEBG | oximeter: contacting nexus
|
| 55 | 2024-02-01T08:59:14.778Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 56 | 2024-02-01T08:59:14.778Z | WARN | oximeter: failed to contact nexus, will retry in 4.365400991s
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 57 | 2024-02-01T08:59:19.147Z | DEBG | oximeter: contacting nexus
|
| 58 | 2024-02-01T08:59:19.147Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 59 | 2024-02-01T08:59:19.148Z | WARN | oximeter: failed to contact nexus, will retry in 23.684875372s
error = "error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = oximeter/collector/src/lib.rs:271
|
| 60 | 2024-02-01T08:59:42.844Z | DEBG | oximeter: contacting nexus
|
| 61 | 2024-02-01T08:59:42.844Z | DEBG | oximeter (DnsResolver): lookup_ipv6 srv
dns_name = _nexus._tcp.control-plane.oxide.internal
|
| 62 | 2024-02-01T08:59:42.914Z | INFO | oximeter: oximeter registered with nexus
file = oximeter/collector/src/lib.rs:285
id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
|
| 63 | 2024-02-01T08:59:43.890Z | INFO | oximeter (dropshot): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::d]:12223
remote_addr = [fd00:1122:3344:101::b]:44970
|
| 64 | 2024-02-01T08:59:43.918Z | DEBG | oximeter (oximeter-agent): registered new metric producer
address = [fd00:1122:3344:101::b]:12221
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
producer_id = 9ea7e9a1-87b6-4cf5-a153-6d7a1eb86b9c
|
| 65 | 2024-02-01T08:59:43.920Z | INFO | oximeter (dropshot): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 22826
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
remote_addr = [fd00:1122:3344:101::b]:44970
req_id = 2d2dffba-9e17-4b60-b791-03cde92c1e42
response_code = 204
uri = /producers
|
| 66 | 2024-02-01T08:59:43.956Z | DEBG | oximeter (collection-task): starting oximeter collection task
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
interval = 10s
producer_id = 9ea7e9a1-87b6-4cf5-a153-6d7a1eb86b9c
|
| 67 | 2024-02-01T08:59:53.961Z | DEBG | oximeter (collection-task): collecting from producer
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
producer_id = 9ea7e9a1-87b6-4cf5-a153-6d7a1eb86b9c
|
| 68 | 2024-02-01T08:59:53.987Z | DEBG | oximeter (collection-task): collected results from producer
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
n_results = 4
producer_id = 9ea7e9a1-87b6-4cf5-a153-6d7a1eb86b9c
|
| 69 | 2024-02-01T08:59:58.303Z | DEBG | oximeter (results-sink): inserting 61 samples into database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
|
| 70 | 2024-02-01T08:59:58.303Z | DEBG | oximeter (clickhouse-client): unrolling 61 total samples
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 71 | 2024-02-01T08:59:58.329Z | DEBG | oximeter (clickhouse-client): retrieving timeseries schema from database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 72 | 2024-02-01T08:59:58.389Z | DEBG | oximeter (clickhouse-client): new timeseries schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
schema = {"timeseries_name":"http_service:request_latency_histogram","fields.name":["id","method","name","route","status_code"],"fields.type":["Uuid","String","String","String","I64"],"fields.source":["Target","Metric","Target","Metric","Metric"],"datum_type":"HistogramF64","created":"2024-02-01 08:59:58.329723442"}
timeseries_name = http_service:request_latency_histogram
|
| 73 | 2024-02-01T08:59:58.441Z | DEBG | oximeter (clickhouse-client): retrieving timeseries schema from database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 74 | 2024-02-01T08:59:58.468Z | DEBG | oximeter (clickhouse-client): new timeseries schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
schema = {"timeseries_name":"collection_target:cpus_provisioned","fields.name":["id"],"fields.type":["Uuid"],"fields.source":["Target"],"datum_type":"I64","created":"2024-02-01 08:59:58.440998867"}
timeseries_name = collection_target:cpus_provisioned
|
| 75 | 2024-02-01T08:59:58.468Z | DEBG | oximeter (clickhouse-client): retrieving timeseries schema from database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 76 | 2024-02-01T08:59:58.472Z | DEBG | oximeter (clickhouse-client): new timeseries schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
schema = {"timeseries_name":"collection_target:ram_provisioned","fields.name":["id"],"fields.type":["Uuid"],"fields.source":["Target"],"datum_type":"I64","created":"2024-02-01 08:59:58.468303436"}
timeseries_name = collection_target:ram_provisioned
|
| 77 | 2024-02-01T08:59:58.472Z | DEBG | oximeter (clickhouse-client): retrieving timeseries schema from database
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 78 | 2024-02-01T08:59:58.474Z | DEBG | oximeter (clickhouse-client): new timeseries schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
schema = {"timeseries_name":"collection_target:virtual_disk_space_provisioned","fields.name":["id"],"fields.type":["Uuid"],"fields.source":["Target"],"datum_type":"I64","created":"2024-02-01 08:59:58.472517552"}
timeseries_name = collection_target:virtual_disk_space_provisioned
|
| 79 | 2024-02-01T08:59:58.475Z | DEBG | oximeter (clickhouse-client): inserting 4 new timeseries schema
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
|
| 80 | 2024-02-01T08:59:58.497Z | DEBG | oximeter (clickhouse-client): inserted rows into table
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
n_rows = 40
table_name = oximeter.fields_i64
|
| 81 | 2024-02-01T08:59:58.499Z | DEBG | oximeter (clickhouse-client): inserted rows into table
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
n_rows = 120
table_name = oximeter.fields_string
|
| 82 | 2024-02-01T08:59:58.509Z | DEBG | oximeter (clickhouse-client): inserted rows into table
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
n_rows = 55
table_name = oximeter.fields_uuid
|
| 83 | 2024-02-01T08:59:58.518Z | DEBG | oximeter (clickhouse-client): inserted rows into table
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
n_rows = 40
table_name = oximeter.measurements_histogramf64
|
| 84 | 2024-02-01T08:59:58.525Z | DEBG | oximeter (clickhouse-client): inserted rows into table
collector_id = e9ef6757-f42f-4ae5-8548-fb6ab94c30e9
id = d32c82d3-8a43-414e-920e-9f46d93a0222
n_rows = 21
table_name = oximeter.measurements_i64
|