[ Aug 31 16:27:43 Disabled. ] [ Aug 31 16:27:43 Rereading configuration. ] [ Aug 31 16:27:44 Enabled. ] [ Aug 31 16:27:44 Rereading configuration. ] [ Aug 31 16:27:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Aug 31 16:27:44 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:44.78501593Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:44.807514935Z","hostname":"oxz_switch","pid":2293} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-08-31T16:27:44.807563686Z","hostname":"oxz_switch","pid":2293} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:44.807577427Z","hostname":"oxz_switch","pid":2293} {"msg":"admin: listening on [::]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:44.80920616Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:45.807569852Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:46.807623629Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:47.807733467Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:48.807784684Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:49.807902493Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:50.807977992Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:51.80808942Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:52.80823714Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:53.808276627Z","hostname":"oxz_switch","pid":2293} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:27:54.808337184Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sm initialized with addr fe80::d096:edff:fe57:65ab on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:55.808447933Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:27:55.808475874Z","hostname":"oxz_switch","pid":2293} [ Aug 31 16:27:55 Rereading configuration. ] [ Aug 31 16:27:55 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 212"). ] + CONTRACT='' + getopts c: o + CONTRACT=212 + getopts c: o + [[ -n 212 ]] + pgrep -c 212 + PARENT_PID=2289 + [[ -n 2289 ]] + pgrep -P 2289 + CHILD_PID=2293 + [[ -n 2293 ]] + kill -USR1 2293 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:55.943169713Z","hostname":"oxz_switch","pid":2293} [ Aug 31 16:27:55 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:27:55.944992722Z","hostname":"oxz_switch","pid":2293} Aug 31 16:27:55.945 DEBG registered DTrace probes Aug 31 16:27:55.946 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 31 16:27:55.946 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Aug 31 16:27:55.946 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 31 16:27:55.946 DEBG Nexus IP not provided, will use DNS to resolve it Aug 31 16:27:55.946 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Aug 31 16:27:55.946 DEBG starting producer registration task Aug 31 16:27:55.946 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: b7c19bcd-9a6a-4d3c-8ca9-de4a2fcab9e2, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Aug 31 16:27:55.946 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task Aug 31 16:27:55.948 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 781, tv_nsec: 44109184 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:27:55.948 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:27:55.949 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"could not list multicast groups to seed sweep, retrying later: Communication Error: error sending request for url (http://localhost:12224/multicast/groups)","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:04.813536072Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] nbr is fe80::141c:4bff:fe9e:68d8@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.809761088Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:07.809798999Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] exchange: listening on [fe80::d096:edff:fe57:65ab]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.809987274Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.810276402Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.81278552Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:07.81281387Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.813985732Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.814933527Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.815247296Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.815480072Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.815688127Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:07.816028507Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:07.816731975Z","hostname":"oxz_switch","pid":2293} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fdc5:8c1d:5afb:16e7::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:09.993707361Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:10.951 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:28:10.951 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 15.004234331s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Aug 31 16:28:11.103 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 796, tv_nsec: 198996992 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:28:11.103 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:28:11.104 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.157631406Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:18.157691158Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:18.157753649Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.158678474Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.159304241Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.159487806Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.159687621Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.159858626Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:18.160081332Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:20.15788284Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:22.157904101Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:24.157958671Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:26.158002377Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:26.426 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:28:26.426 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 30.159375615s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Aug 31 16:28:26.983 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 811, tv_nsec: 761702146 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:28:26.984 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:28:26.985 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:28.158059966Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.159102004Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:28.159121034Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.160106401Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.160660426Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.160911793Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.161090067Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.161327074Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:28.161505138Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:30.158113213Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:32.158155529Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:34.158215389Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:36.158254878Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:38.158308027Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.159850198Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:38.159874269Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.160759933Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.1614172Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.161687927Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.161817121Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.161996256Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:38.162245042Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:40.158385556Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:41.945 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:28:41.945 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 45.721443183s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:42.158419155Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:42.755 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 827, tv_nsec: 576794742 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:28:42.756 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:28:42.756 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:44.158496705Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:46.158522203Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:48.158585022Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.160690788Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:48.160718349Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.161607963Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.162290962Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.162458476Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.162671192Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.162839092Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:48.163080213Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:50.15864397Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:52.158701009Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:54.158763328Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:56.158774075Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:57.716 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:28:57.717 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 61.536192047s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:28:58.158808293Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.161606788Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:28:58.161655839Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.162549734Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.163132529Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.163308074Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.163607892Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.163746876Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:28:58.163921391Z","hostname":"oxz_switch","pid":2293} Aug 31 16:28:58.752 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 843, tv_nsec: 616971023 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:28:58.752 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:28:58.753 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:00.159807337Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:02.160842563Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:04.161861686Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:06.16286844Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.162404524Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:08.162430265Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.163986237Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.164695906Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.164915202Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.165153359Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.165361894Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:08.165573214Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:12.164986617Z","hostname":"oxz_switch","pid":2293} Aug 31 16:29:13.718 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:29:13.719 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 77.576650425s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.163296394Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:18.163353213Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.164157985Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.164815795Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.165077702Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.165217566Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.165369251Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:18.165653546Z","hostname":"oxz_switch","pid":2293} Aug 31 16:29:19.284 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 864, tv_nsec: 183998581 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:29:19.285 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:29:19.286 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.163869077Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:28.163950909Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.165005897Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.165909701Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.166063205Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.166222329Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.166386234Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:28.166541618Z","hostname":"oxz_switch","pid":2293} Aug 31 16:29:34.289 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:29:34.289 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 98.143892143s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::141c:4bff:fe9e:68d8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-31T16:29:40.088576134Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.315121502Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:40.315145793Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.339747151Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.361263459Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.361745642Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.362001538Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.362177973Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:40.36242975Z","hostname":"oxz_switch","pid":2293} Aug 31 16:29:40.611 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2de03509-a130-46f9-b202-200f0008608c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("97eb4bac-a1da-4d58-bc01-6d83ce489de8.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ed39db1d-f9a4-4481-98a4-d08a7faf8f62.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 885, tv_nsec: 482763420 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 31 16:29:40.615 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 31 16:29:40.627 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.692032188Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:49.693373215Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.753695155Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.776852627Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.778478201Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.778906152Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.780763842Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:49.783617779Z","hostname":"oxz_switch","pid":2293} Aug 31 16:29:55.645 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 31 16:29:55.650 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 119.501018387s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.616990744Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-31T16:29:59.617027885Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.618073893Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.619283385Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.619581453Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.61981262Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::141c:4bff:fe9e:68d8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.620119628Z","hostname":"oxz_switch","pid":2293} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-31T16:29:59.620391555Z","hostname":"oxz_switch","pid":2293}