[ Jul 4 15:27:57 Disabled. ] [ Jul 4 15:27:57 Rereading configuration. ] [ Jul 4 15:27:57 Enabled. ] [ Jul 4 15:27:57 Rereading configuration. ] [ Jul 4 15:27:57 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Jul 4 15:27:57 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:27:57.92103182Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:27:57.94424815Z","hostname":"oxz_switch","pid":2296} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-07-04T15:27:57.9442972Z","hostname":"oxz_switch","pid":2296} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:27:57.94430687Z","hostname":"oxz_switch","pid":2296} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:27:57.945211605Z","hostname":"oxz_switch","pid":2296} {"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:27:57.945777138Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:27:58.944348603Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:27:59.944414925Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:00.944503257Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:01.94458075Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:02.944766843Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:03.945587549Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:04.945666982Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:05.945754094Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:06.945865527Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:07.946136721Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sm initialized with addr fe80::b828:78ff:fe76:e0dc on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:08.952390618Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-07-04T15:28:08.952448798Z","hostname":"oxz_switch","pid":2296} [ Jul 4 15:28:09 Rereading configuration. ] [ Jul 4 15:28:09 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2293 + [[ -n 2293 ]] + pgrep -P 2293 + CHILD_PID=2296 + [[ -n 2296 ]] + kill -USR1 2296 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:09.544356464Z","hostname":"oxz_switch","pid":2296} [ Jul 4 15:28:09 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:09.546441425Z","hostname":"oxz_switch","pid":2296} Jul 04 15:28:09.547 DEBG registered DTrace probes Jul 04 15:28:09.547 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 04 15:28:09.548 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Jul 04 15:28:09.548 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 04 15:28:09.548 DEBG Nexus IP not provided, will use DNS to resolve it Jul 04 15:28:09.548 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Jul 04 15:28:09.548 DEBG starting producer registration task Jul 04 15:28:09.548 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 83731314-6790-4eca-8572-1f46a0f641a8, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Jul 04 15:28:09.548 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fd7c:ec0:3421:5e83::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:10.769035204Z","hostname":"oxz_switch","pid":2296} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd7c:ec0:3421:5e83::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.769342866Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fd7c:ec0:3421:5e83::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:10.769416776Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] nbr is fe80::1c3e:b8ff:fe52:1245@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.953102116Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-07-04T15:28:10.953190666Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] exchange: listening on [fe80::b828:78ff:fe76:e0dc]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.953401607Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.953537678Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.95565333Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-07-04T15:28:10.95568266Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::1c3e:b8ff:fe52:1245 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.957610551Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::1c3e:b8ff:fe52:1245 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.959133599Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::1c3e:b8ff:fe52:1245 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.959511012Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::1c3e:b8ff:fe52:1245 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.959731263Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::1c3e:b8ff:fe52:1245 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.960032104Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-04T15:28:10.960312296Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-07-04T15:28:10.96111451Z","hostname":"oxz_switch","pid":2296} Jul 04 15:28:14.551 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65300, tv_nsec: 102025325 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:28:14.551 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:28:14.552 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 04 15:28:30.152 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:28:30.152 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 20.005903564s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 04 15:28:30.304 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65315, tv_nsec: 257620767 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:28:30.304 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:28:30.305 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:31.55263938Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:33.552691505Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:35.552764949Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:37.552820354Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:39.552866199Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:41.552911564Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:43.552938999Z","hostname":"oxz_switch","pid":2296} Jul 04 15:28:45.265 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:28:45.265 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 35.160178398s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:45.552999483Z","hostname":"oxz_switch","pid":2296} Jul 04 15:28:45.757 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65330, tv_nsec: 752983211 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:28:45.758 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:28:45.759 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:47.553017517Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:49.553045371Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:51.553107645Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:53.5531573Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:55.553202795Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:57.5535598Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:28:59.553613854Z","hostname":"oxz_switch","pid":2296} Jul 04 15:29:00.720 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:29:00.720 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 50.656055755s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:01.553657389Z","hostname":"oxz_switch","pid":2296} Jul 04 15:29:01.670 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65346, tv_nsec: 708633735 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:29:01.671 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:29:01.672 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:03.553695743Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:05.553739288Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:07.553780802Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:09.553828866Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:11.553885811Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:13.553919704Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:15.55398257Z","hostname":"oxz_switch","pid":2296} Jul 04 15:29:16.632 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:29:16.632 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 66.611272907s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:17.554029284Z","hostname":"oxz_switch","pid":2296} Jul 04 15:29:19.112 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65364, tv_nsec: 197502328 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:29:19.113 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:29:19.114 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:19.554079418Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:21.554109143Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:23.554189667Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:25.554206462Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:27.554287796Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::1c3e:b8ff:fe52:1245 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-04T15:29:29.554340101Z","hostname":"oxz_switch","pid":2296} Jul 04 15:29:34.074 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:29:34.074 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 84.10023473s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 04 15:29:36.821 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65381, tv_nsec: 953689048 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:29:36.821 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:29:36.822 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 04 15:29:51.783 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 04 15:29:51.783 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 101.856304689s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 04 15:30:01.700 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2247edfc-d32b-4309-8cda-be36ebf10233.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3d208fbf-c603-4fb6-b206-b2ea1ca61e22.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("421e18dc-6086-4481-8eec-c85164e55278.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 65406, tv_nsec: 882858631 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 04 15:30:01.701 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Jul 04 15:30:01.703 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task