[ Apr 7 18:41:09 Disabled. ] [ Apr 7 18:41:09 Rereading configuration. ] [ Apr 7 18:41:10 Enabled. ] [ Apr 7 18:41:10 Rereading configuration. ] [ Apr 7 18:41:10 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Apr 7 18:41:10 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:10.74233399Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:10.763947714Z","hostname":"oxz_switch","pid":2285} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-04-07T18:41:10.764001136Z","hostname":"oxz_switch","pid":2285} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:10.764016526Z","hostname":"oxz_switch","pid":2285} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:10.765137805Z","hostname":"oxz_switch","pid":2285} {"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:10.765639158Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:11.764128171Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:12.764214825Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:13.76432681Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:14.764423634Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:15.764540899Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:16.764649634Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:17.7647916Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:18.764956236Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:19.765059421Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:20.765163926Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:21.765231709Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:22.765302243Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:23.765395627Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:24.77324023Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:25.766643393Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:26.766728767Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:27.766810552Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:28.766864115Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:29.766948419Z","hostname":"oxz_switch","pid":2285} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:30.767027743Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] sm initialized with addr fe80::cc27:89ff:fe40:bc36 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:31.767127407Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-04-07T18:41:31.76722212Z","hostname":"oxz_switch","pid":2285} [ Apr 7 18:41:32 Rereading configuration. ] [ Apr 7 18:41:32 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 212"). ] + CONTRACT='' + getopts c: o + CONTRACT=212 + getopts c: o + [[ -n 212 ]] + pgrep -c 212 + PARENT_PID=2284 + [[ -n 2284 ]] + pgrep -P 2284 + CHILD_PID=2285 + [[ -n 2285 ]] + kill -USR1 2285 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:32.210050786Z","hostname":"oxz_switch","pid":2285} [ Apr 7 18:41:32 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:32.212957771Z","hostname":"oxz_switch","pid":2285} Apr 07 18:41:32.213 DEBG registered DTrace probes Apr 07 18:41:32.214 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Apr 07 18:41:32.214 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328 Apr 07 18:41:32.214 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Apr 07 18:41:32.214 DEBG Nexus IP not provided, will use DNS to resolve it Apr 07 18:41:32.214 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/internal-dns/resolver/src/resolver.rs:144 Apr 07 18:41:32.214 DEBG starting producer registration task Apr 07 18:41:32.214 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 682b272f-5b9a-4112-8a9f-378a5d55be7a, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:281 Apr 07 18:41:32.214 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task Apr 07 18:41:32.215 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 1.037927ms, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:41:32.530 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 668, tv_nsec: 12610730 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:41:32.531 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:41:32.532 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fddb:532e:9bf8:e76b::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:33.423742533Z","hostname":"oxz_switch","pid":2285} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fddb:532e:9bf8:e76b::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.423934728Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fddb:532e:9bf8:e76b::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:33.424051711Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] nbr is fe80::ce:83ff:fe3e:ffd0@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.76782868Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-04-07T18:41:33.767941973Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] exchange: listening on [fe80::cc27:89ff:fe40:bc36]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.768152658Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.768307502Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.770672663Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-04-07T18:41:33.770690193Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::ce:83ff:fe3e:ffd0 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.771394001Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::ce:83ff:fe3e:ffd0 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.772656413Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::ce:83ff:fe3e:ffd0 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.772991802Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::ce:83ff:fe3e:ffd0 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.773227328Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::ce:83ff:fe3e:ffd0 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.773597917Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-04-07T18:41:33.773863454Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-04-07T18:41:33.773875555Z","hostname":"oxz_switch","pid":2285} Apr 07 18:41:47.394 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:41:47.395 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 15.318238874s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:41:47.687 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 683, tv_nsec: 308239324 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:41:47.688 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:41:47.688 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:51.629992064Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:53.630268989Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:55.630290813Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:57.630332815Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:41:59.630377949Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:01.630417013Z","hostname":"oxz_switch","pid":2285} Apr 07 18:42:02.648 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:42:02.648 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 30.612791141s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:03.630433736Z","hostname":"oxz_switch","pid":2285} Apr 07 18:42:04.024 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 699, tv_nsec: 689958137 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:42:04.025 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:42:04.025 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:05.631435994Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:07.631495719Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:09.63177556Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:11.632546282Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:13.632570206Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:15.63260795Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:17.632641584Z","hostname":"oxz_switch","pid":2285} Apr 07 18:42:18.986 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:42:18.986 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 46.995652643s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:19.63266416Z","hostname":"oxz_switch","pid":2285} Apr 07 18:42:20.258 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 715, tv_nsec: 968082265 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:42:20.259 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:42:20.259 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:21.633747081Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:23.633731365Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:25.633734579Z","hostname":"oxz_switch","pid":2285} {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:42:27.633797963Z","hostname":"oxz_switch","pid":2285} Apr 07 18:42:35.220 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:42:35.220 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 63.273471353s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:42:40.025 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 735, tv_nsec: 788219286 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:42:40.025 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:42:40.025 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 07 18:42:54.986 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:42:54.986 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 83.093409199s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:43:06.352 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 762, tv_nsec: 187200861 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:43:06.353 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:43:06.353 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 07 18:43:21.313 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:43:21.313 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 109.492181619s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:43:42.746 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 798, tv_nsec: 679561268 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:43:42.746 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:43:42.746 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 07 18:43:57.717 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:43:57.717 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 145.985375276s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 Apr 07 18:44:20.115 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("189420b8-8e87-4144-950f-5019d1146431.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9654aa96-0cfa-4537-ad68-c75faa53f0da.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bc51defb-f635-4b22-bce5-7bb30851da57.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 836, tv_nsec: 79575280 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 07 18:44:20.116 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 07 18:44:20.118 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 07 18:44:35.119 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 07 18:44:35.120 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 183.388077048s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79fac7d/oximeter/producer/src/lib.rs:377 {"msg":"[tfportrear0_0] neighbor buskin@fe80::ce:83ff:fe3e:ffd0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-07T18:44:56.484887332Z","hostname":"oxz_switch","pid":2285}