[ Oct 21 22:14:48 Enabled. ]
[ Oct 21 22:14:48 Rereading configuration. ]
[ Oct 21 22:14:49 Rereading configuration. ]
[ Oct 21 22:14:51 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 0bffbf95-986c-4630-a1a6-9704dbf12753 &"). ]
[ Oct 21 22:14:51 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:51.397259191Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:51.420356973Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:233"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:51.428987032Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:51.783705126Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"97e2e148-afc9-4380-bfb1-17ea8279a88b","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:887","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:51.783804998Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:297","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)\"))","retry_after":"174.340777ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:51.9585177Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:52.576841661Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"aa6609a8-05fd-4d01-8f5b-61bf773b4446","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:887","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:52.576901403Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:297","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)\"))","retry_after":"306.084531ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:52.883366897Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"failed to read version","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:53.135396238Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"3a72a197-d3e3-4d2b-bb58-9676a18fee6c","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:887","error":"Telemetry database unavailable: error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:53.13544544Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:297","error":"Database(DatabaseUnavailable(\"error sending request for url (http://[fd00:1122:3344:101::e]:8123/?output_format_json_quote_64bit_integers=0&wait_end_of_query=1)\"))","retry_after":"1.467516621s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:54.604145645Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:54.706993562Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:883"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:54.707029002Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:54.707036173Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:821"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:54.708710177Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:883"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:54.708722218Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:825","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:54.708730688Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:830"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:54.709395145Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:54.712937499Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:56.46209862Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"id":"ec103531-39ec-4c68-9ae7-66e89b625652","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"0bffbf95-986c-4630-a1a6-9704dbf12753","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:853","version":12}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625268765Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625311966Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625321897Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625333117Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2024-10-21T22:14:56.625342937Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625353157Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:56.625373868Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:57.354946429Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:57.355269637Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 256.824339ms","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:57.355285938Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:57.61337194Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:57.613447322Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:57.613597416Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 735.076449ms","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:57.613609756Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:58.34971161Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:58.349807452Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:58.349958656Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.091670794s","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:58.349985807Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:59.443113336Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:59.443211339Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:14:59.443391244Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.981657664s","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:14:59.443413034Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:15:01.425476509Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:15:01.425560411Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-21T22:15:01.425708395Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 3.185989405s","v":0,"name":"oximeter","level":40,"time":"2024-10-21T22:15:01.425767516Z","hostname":"oxz_oximeter_0bffbf95-986c-4630-a1a6-9704dbf12753","pid":21350,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}