[ Dec 20 01:29:30 Enabled. ]
[ Dec 20 01:29:30 Rereading configuration. ]
[ Dec 20 01:29:32 Rereading configuration. ]
[ Dec 20 01:29:38 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id a632771f-6a97-4f85-8047-76cfea999e76 &"). ]
[ Dec 20 01:29:38 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:39.131370651Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:39.142187974Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:264"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:39.146339801Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:39.237599622Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:42.189837553Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"oximeter database does not exist, creating","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.189886734Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent"}
{"msg":"reading db version","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:42.189896095Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:800"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:42.191036696Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:900"}
{"msg":"read oximeter database version","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:42.191054356Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:805","version":0}
{"msg":"initializing oximeter schema","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:42.191062176Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:810"}
{"msg":"initializing ClickHouse database","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.191070327Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent"}
{"msg":"inserting current version","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:42.535123333Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"f78651cf-374c-4146-af15-46cc4d618ead","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:831","version":14}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.576117101Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"id":"5890e888-9272-4cda-bd3d-4f67dd8486f6","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.610570487Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.610600628Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.610615368Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.610624098Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:42.610633978Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.6/src/server.rs:328"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.624910246Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.624962186Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.624982287Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.822881335Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:42.828261986Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 233.631201ms","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:42.828334107Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.063042385Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.063167237Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.06334103Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 308.183692ms","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:43.063417332Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.372854348Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.37294013Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:43.373210644Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 1.486699262s","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:43.373257345Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:44.86031269Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:44.860388492Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:44.860550245Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/collectors\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to contact nexus, will retry in 2.804987973s","v":0,"name":"oximeter","level":40,"time":"2025-12-20T01:29:44.860601096Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:409","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/collectors)\""}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:47.667491278Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:47.667679851Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:47.880939675Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"fc92f2bd-2506-4b09-b2c6-b38eae035b92\", \"date\": \"Sat, 20 Dec 2025 01:29:47 GMT\"} })"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:47.884576313Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"file":"oximeter/collector/src/lib.rs:427","id":"a632771f-6a97-4f85-8047-76cfea999e76"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:47.887804844Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:554"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:47.888863324Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:557"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:47.897946964Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"body":"None","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/a632771f-6a97-4f85-8047-76cfea999e76/producers?sort_by=id_ascending","method":"GET"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2025-12-20T01:29:47.942992087Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/collectors/a632771f-6a97-4f85-8047-76cfea999e76/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c7ecb9d-1f81-41ef-9f2e-7b138b479ed8\", \"content-length\": \"29\", \"date\": \"Sat, 20 Dec 2025 01:29:47 GMT\"} })"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2025-12-20T01:29:47.94312354Z","hostname":"oxz_oximeter_a632771f-6a97-4f85-8047-76cfea999e76","pid":24607,"collector_ip":"fd00:1122:3344:101::d","collector_id":"a632771f-6a97-4f85-8047-76cfea999e76","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:632","n_current_tasks":0,"n_pruned_tasks":0}