[ Nov 26 05:23:50 Enabled. ]
[ Nov 26 05:23:50 Rereading configuration. ]
[ Nov 26 05:23:52 Rereading configuration. ]
[ Nov 26 05:23:55 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id a4232f21-0674-457e-97a4-41c6f360d0c1 &"). ]
[ Nov 26 05:23:55 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:56.075310531Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:56.07724179Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"file":"oximeter/collector/src/lib.rs:227"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:56.078504555Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:56.092857545Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-11-26T05:23:57.951076071Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:823"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:57.951123561Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:57.951133752Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:732"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2024-11-26T05:23:57.952006549Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:823"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:57.95202112Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:736","version":0}
{"msg":"wiping and re-initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:57.95202973Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:741"}
{"msg":"wiping ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:57.95204546Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:57.954091371Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:58.143610895Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"id":"1ad8802b-a62c-4b21-8a22-f169a1cd1c4a","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:764","version":13}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179352005Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179388846Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179406957Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:23:58.179415087Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179424297Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179448867Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.179471238Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.313069213Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.314819228Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 243.949001ms","v":0,"name":"oximeter","level":40,"time":"2024-11-26T05:23:58.314842509Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"file":"oximeter/collector/src/lib.rs:356","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.559957744Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.560044935Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:58.56025485Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 603.019041ms","v":0,"name":"oximeter","level":40,"time":"2024-11-26T05:23:58.56027596Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"file":"oximeter/collector/src/lib.rs:356","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:59.164613631Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:59.164682003Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:23:59.164914277Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.071411341s","v":0,"name":"oximeter","level":40,"time":"2024-11-26T05:23:59.164929268Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"file":"oximeter/collector/src/lib.rs:356","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:24:00.238022685Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:24:00.238109287Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:24:00.285261588Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"06bec090-45f5-42a4-b4b6-efdebe4ec6eb\", \"date\": \"Tue, 26 Nov 2024 05:24:00 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:24:00.285383381Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"file":"oximeter/collector/src/lib.rs:374","id":"a4232f21-0674-457e-97a4-41c6f360d0c1"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2024-11-26T05:24:00.286491823Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:1091"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:24:00.288782899Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/a4232f21-0674-457e-97a4-41c6f360d0c1/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2024-11-26T05:24:00.289037564Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/a4232f21-0674-457e-97a4-41c6f360d0c1/producers?sort_by=id_ascending\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"error fetching next assigned producer, abandoning this refresh attempt","v":0,"name":"oximeter","level":50,"time":"2024-11-26T05:24:00.289063955Z","hostname":"oxz_oximeter_a4232f21-0674-457e-97a4-41c6f360d0c1","pid":24212,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a4232f21-0674-457e-97a4-41c6f360d0c1","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:1120","err":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/collectors/a4232f21-0674-457e-97a4-41c6f360d0c1/producers?sort_by=id_ascending)"}