[ Oct 15 23:29:41 Enabled. ]
[ Oct 15 23:29:41 Rereading configuration. ]
[ Oct 15 23:29:42 Rereading configuration. ]
[ Oct 15 23:29:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id d840e5b1-0bc2-44e7-bb14-a2548552ccff &"). ]
[ Oct 15 23:29:44 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:44.683211136Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:44.686978946Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:252"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:44.689050969Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468002067Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468054728Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468070458Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468080168Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:47.468090599Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468099869Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.468108969Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.606500169Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.606830898Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 183.885666ms","v":0,"name":"oximeter","level":40,"time":"2024-10-15T23:29:47.606849415Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.792336002Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.792413751Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:47.792616152Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 381.052765ms","v":0,"name":"oximeter","level":40,"time":"2024-10-15T23:29:47.792635343Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:48.174986616Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:48.175047528Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:48.175197931Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.482693727s","v":0,"name":"oximeter","level":40,"time":"2024-10-15T23:29:48.175221291Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:49.659226653Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:49.659287834Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:49.659501508Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.955866565s","v":0,"name":"oximeter","level":40,"time":"2024-10-15T23:29:49.659517869Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:51.6165008Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:51.616555921Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:51.616680233Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 3.564701455s","v":0,"name":"oximeter","level":40,"time":"2024-10-15T23:29:51.616698514Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:373","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:55.182400116Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:55.182466277Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:55.244480332Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"eb9889ff-24ae-4f5b-9c63-641787e01d53\", \"date\": \"Tue, 15 Oct 2024 23:29:55 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:55.245063114Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"file":"oximeter/collector/src/lib.rs:391","id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:55.246173747Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:761"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:55.247991075Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/d840e5b1-0bc2-44e7-bb14-a2548552ccff/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:55.287234541Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/d840e5b1-0bc2-44e7-bb14-a2548552ccff/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f98d4020-4754-4ada-ab84-1a115493612b\", \"content-length\": \"29\", \"date\": \"Tue, 15 Oct 2024 23:29:55 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:55.287912485Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:809","n_current_tasks":0,"n_pruned_tasks":0}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:57.819565344Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33248"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:57.820046464Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","address":"[fd00:1122:3344:101::1]:64843","producer_id":"30e33352-114a-4c01-998c-ae8a42d33861"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:57.820092055Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"uri":"/producers","method":"POST","req_id":"98cd6ead-6f25-482b-ba3d-b0c322807ced","remote_addr":"[fd00:1122:3344:101::c]:33248","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":402,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:57.870914645Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"address":"[fd00:1122:3344:101::1]:64843","producer_id":"30e33352-114a-4c01-998c-ae8a42d33861","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","interval":"30s"}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:57.99806953Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60293"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:57.998415477Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:49672","producer_id":"aa6deb4c-b0e2-4863-b616-9066f7feb811"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:57.998485299Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"uri":"/producers","method":"POST","req_id":"3b47f32f-fc0d-4c9e-b2a1-5c99c4386811","remote_addr":"[fd00:1122:3344:101::c]:60293","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":353,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:58.037149232Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"address":"[fd00:1122:3344:101::2]:49672","producer_id":"aa6deb4c-b0e2-4863-b616-9066f7feb811","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","interval":"10s"}
{"msg":"accepted connection","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:59.203013761Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53200"}
{"msg":"registered new metric producer","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:59.203126424Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","address":"[fd00:1122:3344:101::2]:8001","producer_id":"ce11462b-8724-45a2-9809-d4094fdcf999"}
{"msg":"request completed","v":0,"name":"oximeter","level":30,"time":"2024-10-15T23:29:59.203187535Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"uri":"/producers","method":"POST","req_id":"e376bc1d-288d-4915-b675-1911225f3ab2","remote_addr":"[fd00:1122:3344:101::c]:53200","local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"204"}
{"msg":"starting oximeter collection task","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:29:59.236593648Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"address":"[fd00:1122:3344:101::2]:8001","producer_id":"ce11462b-8724-45a2-9809-d4094fdcf999","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","interval":"1s"}
{"msg":"collecting from producer","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:30:00.235714001Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"address":"[fd00:1122:3344:101::2]:8001","producer_id":"ce11462b-8724-45a2-9809-d4094fdcf999","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent"}
{"msg":"collected results from producer","v":0,"name":"oximeter","level":20,"time":"2024-10-15T23:30:00.25140225Z","hostname":"oxz_oximeter_d840e5b1-0bc2-44e7-bb14-a2548552ccff","pid":21900,"address":"[fd00:1122:3344:101::2]:8001","producer_id":"ce11462b-8724-45a2-9809-d4094fdcf999","component":"collection-task","collector_ip":"fd00:1122:3344:101::d","collector_id":"d840e5b1-0bc2-44e7-bb14-a2548552ccff","component":"oximeter-agent","n_results":1}