| 1 | | [ Oct 15 23:29:41 Enabled. ] |
| 2 | | [ Oct 15 23:29:41 Rereading configuration. ] |
| 3 | | [ Oct 15 23:29:42 Rereading configuration. ] |
| 4 | | [ Oct 15 23:29:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id d840e5b1-0bc2-44e7-bb14-a2548552ccff &"). ] |
| 5 | | [ Oct 15 23:29:44 Method "start" exited with status 0. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-10-15T23:29:44.683Z | DEBG | oximeter: registered DTrace probes
|
| 8 | 2024-10-15T23:29:44.686Z | INFO | oximeter: starting oximeter server
file = oximeter/collector/src/lib.rs:252
|
| 9 | 2024-10-15T23:29:44.689Z | DEBG | oximeter: creating ClickHouse client
|
| 10 | 2024-10-15T23:29:47.468Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = GET
path = /info
|
| 11 | 2024-10-15T23:29:47.468Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = GET
path = /producers
|
| 12 | 2024-10-15T23:29:47.468Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
path = /producers
|
| 13 | 2024-10-15T23:29:47.468Z | DEBG | oximeter (dropshot): registered endpoint
local_addr = [fd00:1122:3344:101::d]:12223
method = DELETE
path = /producers/{producer_id}
|
| 14 | 2024-10-15T23:29:47.468Z | INFO | oximeter (dropshot): listening
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197
local_addr = [fd00:1122:3344:101::d]:12223
|
| 15 | 2024-10-15T23:29:47.468Z | DEBG | oximeter (dropshot): successfully registered DTrace USDT probes
local_addr = [fd00:1122:3344:101::d]:12223
|
| 16 | 2024-10-15T23:29:47.468Z | DEBG | oximeter: contacting nexus
|
| 17 | 2024-10-15T23:29:47.606Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::b]:12221/metrics/collectors
|
| 18 | 2024-10-15T23:29:47.606Z | DEBG | oximeter: client response
result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/collectors", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) })
|
| 19 | 2024-10-15T23:29:47.606Z | WARN | oximeter: failed to contact nexus, will retry in 183.885666ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)"
file = oximeter/collector/src/lib.rs:373
|
| 20 | 2024-10-15T23:29:47.792Z | DEBG | oximeter: contacting nexus
|
| 21 | 2024-10-15T23:29:47.792Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::c]:12221/metrics/collectors
|
| 22 | 2024-10-15T23:29:47.792Z | DEBG | oximeter: client response
result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/collectors", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) })
|
| 23 | 2024-10-15T23:29:47.792Z | WARN | oximeter: failed to contact nexus, will retry in 381.052765ms
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)"
file = oximeter/collector/src/lib.rs:373
|
| 24 | 2024-10-15T23:29:48.174Z | DEBG | oximeter: contacting nexus
|
| 25 | 2024-10-15T23:29:48.175Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::c]:12221/metrics/collectors
|
| 26 | 2024-10-15T23:29:48.175Z | DEBG | oximeter: client response
result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/collectors", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) })
|
| 27 | 2024-10-15T23:29:48.175Z | WARN | oximeter: failed to contact nexus, will retry in 1.482693727s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)"
file = oximeter/collector/src/lib.rs:373
|
| 28 | 2024-10-15T23:29:49.659Z | DEBG | oximeter: contacting nexus
|
| 29 | 2024-10-15T23:29:49.659Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::a]:12221/metrics/collectors
|
| 30 | 2024-10-15T23:29:49.659Z | DEBG | oximeter: client response
result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/collectors", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) })
|
| 31 | 2024-10-15T23:29:49.659Z | WARN | oximeter: failed to contact nexus, will retry in 1.955866565s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)"
file = oximeter/collector/src/lib.rs:373
|
| 32 | 2024-10-15T23:29:51.616Z | DEBG | oximeter: contacting nexus
|
| 33 | 2024-10-15T23:29:51.616Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::b]:12221/metrics/collectors
|
| 34 | 2024-10-15T23:29:51.616Z | DEBG | oximeter: client response
result = Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/collectors", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) })
|
| 35 | 2024-10-15T23:29:51.616Z | WARN | oximeter: failed to contact nexus, will retry in 3.564701455s
error = "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)"
file = oximeter/collector/src/lib.rs:373
|
| 36 | 2024-10-15T23:29:55.182Z | DEBG | oximeter: contacting nexus
|
| 37 | 2024-10-15T23:29:55.182Z | DEBG | oximeter: client request
body = Some(Body)
method = POST
uri = http://[fd00:1122:3344:101::c]:12221/metrics/collectors
|
| 38 | 2024-10-15T23:29:55.244Z | DEBG | oximeter: client response
result = Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/collectors", status: 204, headers: {"x-request-id": "eb9889ff-24ae-4f5b-9c63-641787e01d53", "date": "Tue, 15 Oct 2024 23:29:55 GMT"} })
|
| 39 | 2024-10-15T23:29:55.245Z | INFO | oximeter: oximeter registered with nexus
file = oximeter/collector/src/lib.rs:391
id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
|
| 40 | 2024-10-15T23:29:55.246Z | INFO | oximeter (oximeter-agent): refreshing list of producers from Nexus
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
file = oximeter/collector/src/agent.rs:761
|
| 41 | 2024-10-15T23:29:55.247Z | DEBG | oximeter: client request
body = None
method = GET
uri = http://[fd00:1122:3344:101::a]:12221/metrics/collectors/d840e5b1-0bc2-44e7-bb14-a2548552ccff/producers?sort_by=id_ascending
|
| 42 | 2024-10-15T23:29:55.287Z | DEBG | oximeter: client response
result = Ok(Response { url: "http://[fd00:1122:3344:101::a]:12221/metrics/collectors/d840e5b1-0bc2-44e7-bb14-a2548552ccff/producers?sort_by=id_ascending", status: 200, headers: {"content-type": "application/json", "x-request-id": "f98d4020-4754-4ada-ab84-1a115493612b", "content-length": "29", "date": "Tue, 15 Oct 2024 23:29:55 GMT"} })
|
| 43 | 2024-10-15T23:29:55.287Z | INFO | oximeter (oximeter-agent): refreshed list of producers from Nexus
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
file = oximeter/collector/src/agent.rs:809
n_current_tasks = 0
n_pruned_tasks = 0
|
| 44 | 2024-10-15T23:29:57.819Z | INFO | oximeter (dropshot): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::d]:12223
remote_addr = [fd00:1122:3344:101::c]:33248
|
| 45 | 2024-10-15T23:29:57.820Z | DEBG | oximeter (oximeter-agent): registered new metric producer
address = [fd00:1122:3344:101::1]:64843
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
producer_id = 30e33352-114a-4c01-998c-ae8a42d33861
|
| 46 | 2024-10-15T23:29:57.820Z | INFO | oximeter (dropshot): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 402
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
remote_addr = [fd00:1122:3344:101::c]:33248
req_id = 98cd6ead-6f25-482b-ba3d-b0c322807ced
response_code = 204
uri = /producers
|
| 47 | 2024-10-15T23:29:57.870Z | DEBG | oximeter (collection-task): starting oximeter collection task
address = [fd00:1122:3344:101::1]:64843
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
interval = 30s
producer_id = 30e33352-114a-4c01-998c-ae8a42d33861
|
| 48 | 2024-10-15T23:29:57.998Z | INFO | oximeter (dropshot): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::d]:12223
remote_addr = [fd00:1122:3344:101::c]:60293
|
| 49 | 2024-10-15T23:29:57.998Z | DEBG | oximeter (oximeter-agent): registered new metric producer
address = [fd00:1122:3344:101::2]:49672
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
producer_id = aa6deb4c-b0e2-4863-b616-9066f7feb811
|
| 50 | 2024-10-15T23:29:57.998Z | INFO | oximeter (dropshot): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 353
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
remote_addr = [fd00:1122:3344:101::c]:60293
req_id = 3b47f32f-fc0d-4c9e-b2a1-5c99c4386811
response_code = 204
uri = /producers
|
| 51 | 2024-10-15T23:29:58.037Z | DEBG | oximeter (collection-task): starting oximeter collection task
address = [fd00:1122:3344:101::2]:49672
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
interval = 10s
producer_id = aa6deb4c-b0e2-4863-b616-9066f7feb811
|
| 52 | 2024-10-15T23:29:59.203Z | INFO | oximeter (dropshot): accepted connection
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
local_addr = [fd00:1122:3344:101::d]:12223
remote_addr = [fd00:1122:3344:101::c]:53200
|
| 53 | 2024-10-15T23:29:59.203Z | DEBG | oximeter (oximeter-agent): registered new metric producer
address = [fd00:1122:3344:101::2]:8001
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
producer_id = ce11462b-8724-45a2-9809-d4094fdcf999
|
| 54 | 2024-10-15T23:29:59.203Z | INFO | oximeter (dropshot): request completed
file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
latency_us = 114
local_addr = [fd00:1122:3344:101::d]:12223
method = POST
remote_addr = [fd00:1122:3344:101::c]:53200
req_id = e376bc1d-288d-4915-b675-1911225f3ab2
response_code = 204
uri = /producers
|
| 55 | 2024-10-15T23:29:59.236Z | DEBG | oximeter (collection-task): starting oximeter collection task
address = [fd00:1122:3344:101::2]:8001
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
interval = 1s
producer_id = ce11462b-8724-45a2-9809-d4094fdcf999
|
| 56 | 2024-10-15T23:30:00.235Z | DEBG | oximeter (collection-task): collecting from producer
address = [fd00:1122:3344:101::2]:8001
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
producer_id = ce11462b-8724-45a2-9809-d4094fdcf999
|
| 57 | 2024-10-15T23:30:00.251Z | DEBG | oximeter (collection-task): collected results from producer
address = [fd00:1122:3344:101::2]:8001
collector_id = d840e5b1-0bc2-44e7-bb14-a2548552ccff
collector_ip = fd00:1122:3344:101::d
n_results = 1
producer_id = ce11462b-8724-45a2-9809-d4094fdcf999
|