[ Nov  3 05:59:37 Enabled. ]
[ Nov  3 05:59:37 Rereading configuration. ]
[ Nov  3 05:59:39 Rereading configuration. ]
[ Nov  3 05:59:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address [fd00:1122:3344:101::d]:12223 --id 1152f3ea-29d3-46c7-a318-208f28faa9c5 &"). ]
[ Nov  3 05:59:44 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"registered DTrace probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:44.304619035Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"starting oximeter server","v":0,"name":"oximeter","level":30,"time":"2026-11-03T05:59:44.305379089Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:291"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:44.306060112Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:44.312747876Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"558ba1fc-4dc3-435b-aa2a-a0b7a8107102","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.054312889Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"558ba1fc-4dc3-435b-aa2a-a0b7a8107102","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.05436178Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.055543846Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"298.246878ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:47.354635104Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:47.354717666Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6c00c19f-057e-4037-9f1b-a78a874bdc31","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.357995747Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6c00c19f-057e-4037-9f1b-a78a874bdc31","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.358033358Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.358045118Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"527.9176ms"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:47.8873321Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:47.887426452Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"20b17a3b-3247-45ef-b0d9-7859e53793df","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.891552319Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"20b17a3b-3247-45ef-b0d9-7859e53793df","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.89159176Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:47.89160335Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"1.27764178s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:49.169940263Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:49.170033064Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"4fc12d57-be5b-4b59-9d39-10360d06f7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:49.177662627Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"4fc12d57-be5b-4b59-9d39-10360d06f7c9","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:49.177701727Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:49.177712858Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"2.186451561s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:51.365088626Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:51.365185288Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6a7367a9-1d7b-4db8-94e5-f613f9b71540","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:51.372336672Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6a7367a9-1d7b-4db8-94e5-f613f9b71540","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:51.372376102Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:51.372387722Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"3.327829208s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:54.701850973Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T05:59:54.702014376Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6db9d086-5d6e-46e3-b776-609a7a3f728f","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"oximeter database does not exist, or is out-of-date","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:54.705899879Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"6db9d086-5d6e-46e3-b776-609a7a3f728f","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/db/src/client/mod.rs:876"}
{"msg":"oximeter database is not present or not at the expected version","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:54.70597111Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:114","error":"Database is not at expected version"}
{"msg":"failed to create ClickHouse client","v":0,"name":"oximeter","level":40,"time":"2026-11-03T05:59:54.706013391Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:343","error":"Database(DatabaseVersionMismatch { expected: 14, found: 0 })","retry_after":"5.773457578s"}
{"msg":"creating ClickHouse client","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.480306093Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.480394255Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"92291d35-cde1-4241-891d-7104492cbe76","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.520398161Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"id":"d49b3489-28f2-4b8e-95ed-1f673a50b233","component":"clickhouse-client","collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.531113971Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/info","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.531197293Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.531207603Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.531252214Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","versions":"all","path":"/producers/{producer_id}","method":"GET"}
{"msg":"listening","v":0,"name":"oximeter","level":30,"time":"2026-11-03T06:00:00.531293185Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.18.0/src/server.rs:339"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.533501036Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"local_addr":"[fd00:1122:3344:101::d]:12223","component":"dropshot"}
{"msg":"registered USDT probes","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.534897842Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"contacting nexus","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.534919662Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.550001083Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors","method":"POST","operation_id":"cpapi_collectors_post"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.572581055Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors\", status: 204, headers: {\"x-request-id\": \"39255175-29b5-4e36-8500-a80ff8b2baca\", \"date\": \"Tue, 03 Nov 2026 06:00:00 GMT\"} })","operation_id":"cpapi_collectors_post"}
{"msg":"oximeter registered with nexus","v":0,"name":"oximeter","level":30,"time":"2026-11-03T06:00:00.572668266Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"file":"oximeter/collector/src/lib.rs:422","id":"1152f3ea-29d3-46c7-a318-208f28faa9c5"}
{"msg":"starting refresh list task","v":0,"name":"oximeter","level":30,"time":"2026-11-03T06:00:00.572684846Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:623"}
{"msg":"refreshing list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-11-03T06:00:00.573765147Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:626"}
{"msg":"client request","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.576284134Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"body":"None","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/1152f3ea-29d3-46c7-a318-208f28faa9c5/producers?sort_by=id_ascending","method":"GET","operation_id":"cpapi_assigned_producers_list"}
{"msg":"client response","v":0,"name":"oximeter","level":20,"time":"2026-11-03T06:00:00.590601271Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"result":"Ok(Response { url: \"http://[fd00:1122:3344:101::a]:12221/metrics/collectors/1152f3ea-29d3-46c7-a318-208f28faa9c5/producers?sort_by=id_ascending\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6dd46e7e-1403-4a99-aa0e-83a2f0664bff\", \"content-length\": \"29\", \"date\": \"Tue, 03 Nov 2026 06:00:00 GMT\"} })","operation_id":"cpapi_assigned_producers_list"}
{"msg":"refreshed list of producers from Nexus","v":0,"name":"oximeter","level":30,"time":"2026-11-03T06:00:00.590699962Z","hostname":"oxz_oximeter_1152f3ea-29d3-46c7-a318-208f28faa9c5","pid":24224,"collector_ip":"fd00:1122:3344:101::d","collector_id":"1152f3ea-29d3-46c7-a318-208f28faa9c5","component":"oximeter-agent","file":"oximeter/collector/src/agent.rs:701","n_current_tasks":0,"n_pruned_tasks":0}