[ Nov 17 01:43:08 Disabled. ] [ Nov 17 01:43:08 Rereading configuration. ] [ Nov 17 01:43:09 Enabled. ] [ Nov 17 01:43:09 Rereading configuration. ] [ Nov 17 01:43:09 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Nov 17 01:43:09 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:09.418281361Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:09.442115264Z","hostname":"oxz_switch","pid":2301} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-11-17T01:43:09.442143485Z","hostname":"oxz_switch","pid":2301} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:09.442196736Z","hostname":"oxz_switch","pid":2301} {"msg":"admin: listening on [::]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:09.443646927Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:10.442244772Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:11.442360669Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:12.442479247Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:13.442648917Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:14.507857563Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:15.445495559Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:16.445572485Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:17.445697413Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:18.445789081Z","hostname":"oxz_switch","pid":2301} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:19.445828017Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] sm initialized with addr fe80::ccb4:80ff:fe1f:1803 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:20.445965005Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-11-17T01:43:20.445993056Z","hostname":"oxz_switch","pid":2301} [ Nov 17 01:43:20 Rereading configuration. ] [ Nov 17 01:43:20 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2296 + [[ -n 2296 ]] + pgrep -P 2296 + CHILD_PID=2301 + [[ -n 2301 ]] + kill -USR1 2301 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:20.641088419Z","hostname":"oxz_switch","pid":2301} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:20.642467659Z","hostname":"oxz_switch","pid":2301} [ Nov 17 01:43:20 Method "refresh" exited with status 0. ] Nov 17 01:43:20.643 DEBG registered DTrace probes Nov 17 01:43:20.643 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Nov 17 01:43:20.643 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Nov 17 01:43:20.644 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Nov 17 01:43:20.644 DEBG Nexus IP not provided, will use DNS to resolve it Nov 17 01:43:20.644 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Nov 17 01:43:20.644 DEBG starting producer registration task Nov 17 01:43:20.644 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: fafa69ba-8ebb-4134-9c8d-32a3dd762af2, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Nov 17 01:43:20.644 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task Nov 17 01:43:25.647 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 964, tv_nsec: 780440291 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:43:25.648 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:43:25.650 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:31.966538674Z","hostname":"oxz_switch","pid":2301,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] nbr is fe80::b061:74ff:fe52:e450@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.446910359Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-11-17T01:43:32.447044852Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] exchange: listening on [fe80::ccb4:80ff:fe1f:1803]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.447312708Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.447478331Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.450156919Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-11-17T01:43:32.450173769Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::b061:74ff:fe52:e450 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.451307903Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::b061:74ff:fe52:e450 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.45252294Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::b061:74ff:fe52:e450 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.452931318Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::b061:74ff:fe52:e450 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.453282996Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::b061:74ff:fe52:e450 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.45349659Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.453823578Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-11-17T01:43:32.454458621Z","hostname":"oxz_switch","pid":2301} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd7c:7c7a:eb14:9aff::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:43:32.972361833Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:38.368461475Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:40.368489535Z","hostname":"oxz_switch","pid":2301} Nov 17 01:43:40.561 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 17 01:43:40.562 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 20.00713138s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 17 01:43:40.717 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 979, tv_nsec: 940026810 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:43:40.717 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:43:40.718 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:42.368530514Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:44.368592369Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:46.368639518Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:48.368659974Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:50.368749812Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:52.368792635Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:54.368859Z","hostname":"oxz_switch","pid":2301} Nov 17 01:43:55.677 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 17 01:43:55.677 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 35.164487592s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 17 01:43:56.234 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 995, tv_nsec: 500028257 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:43:56.235 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:43:56.235 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:56.368890787Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:43:58.368956245Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:00.369011304Z","hostname":"oxz_switch","pid":2301} {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:44:01.82006242Z","hostname":"oxz_switch","pid":2301,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:02.369080013Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:04.369137122Z","hostname":"oxz_switch","pid":2301} Nov 17 01:44:11.195 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 17 01:44:11.196 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 50.724777463s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 17 01:44:12.277 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1011, tv_nsec: 586315876 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:44:12.277 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:44:12.278 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Nov 17 01:44:27.239 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 17 01:44:27.240 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 66.812288958s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 17 01:44:29.040 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1028, tv_nsec: 395583144 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:44:29.041 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:44:29.041 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-17T01:44:31.740301759Z","hostname":"oxz_switch","pid":2301,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} Nov 17 01:44:44.003 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 17 01:44:44.003 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 83.62128715s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 17 01:44:47.793 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("095787c5-85ec-4a47-969b-afb3681bdd56.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2dc16923-126b-4b0f-afc8-395cfcd64db6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcc1f369-db23-4c86-b2ff-9c524640c008.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1047, tv_nsec: 199340554 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 17 01:44:47.794 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Nov 17 01:44:47.795 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:48.37054891Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:52.370608366Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:44:58.370679411Z","hostname":"oxz_switch","pid":2301} {"msg":"[tfportrear0_0] neighbor sock@fe80::b061:74ff:fe52:e450 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-17T01:45:00.370702839Z","hostname":"oxz_switch","pid":2301}