[ Aug 25 09:26:00 Disabled. ] [ Aug 25 09:26:00 Rereading configuration. ] [ Aug 25 09:26:00 Enabled. ] [ Aug 25 09:26:00 Rereading configuration. ] [ Aug 25 09:26:01 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Aug 25 09:26:01 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:01.083256109Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:01.100840713Z","hostname":"oxz_switch","pid":2296} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-08-25T09:26:01.100881734Z","hostname":"oxz_switch","pid":2296} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:01.100896834Z","hostname":"oxz_switch","pid":2296} {"msg":"admin: listening on [::]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:01.102179794Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:02.100931824Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:03.101007325Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:04.101116946Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:05.101223986Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:06.103134224Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:07.103222374Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:08.103301795Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:09.103581828Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:10.103790491Z","hostname":"oxz_switch","pid":2296} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:11.103860731Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sm initialized with addr fe80::cc22:81ff:fe8a:a07 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.103949311Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:12.103973382Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] nbr is fe80::e4a1:b6ff:fee4:954a@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.104605581Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:12.104635221Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] exchange: listening on [fe80::cc22:81ff:fe8a:a07]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.104821504Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.105096688Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:12.106794344Z","hostname":"oxz_switch","pid":2296} [ Aug 25 09:26:12 Rereading configuration. ] [ Aug 25 09:26:12 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2294 + [[ -n 2294 ]] + pgrep -P 2294 + CHILD_PID=2296 + [[ -n 2296 ]] + kill -USR1 2296 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.575629455Z","hostname":"oxz_switch","pid":2296} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:12.57729434Z","hostname":"oxz_switch","pid":2296} [ Aug 25 09:26:12 Method "refresh" exited with status 0. ] Aug 25 09:26:12.584 DEBG registered DTrace probes Aug 25 09:26:12.584 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 25 09:26:12.585 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Aug 25 09:26:12.585 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 25 09:26:12.585 DEBG Nexus IP not provided, will use DNS to resolve it Aug 25 09:26:12.585 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Aug 25 09:26:12.585 DEBG starting producer registration task Aug 25 09:26:12.585 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: fe6f0c88-0ade-4eed-8d21-07b8d5b55aed, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Aug 25 09:26:12.585 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.107994741Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:14.108029412Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.109442443Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.110978076Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.111327691Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.111593485Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.11190042Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.112217254Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:14.113017357Z","hostname":"oxz_switch","pid":2296} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd9a:306e:6720:3c93::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:14.659341263Z","hostname":"oxz_switch","pid":2296} Aug 25 09:26:17.587 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 5.001647502s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Aug 25 09:26:17.746 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2324, tv_nsec: 625530611 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:26:17.747 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:26:17.748 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.108226327Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:22.108259118Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.109275393Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.109898593Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.110079775Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.110218657Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.11036368Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:22.110571403Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:29.459230255Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.451825705Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:31.451868306Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.452759184Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.453397209Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.453592132Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.4537584Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.453925077Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:31.454069849Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:31.459228157Z","hostname":"oxz_switch","pid":2296} Aug 25 09:26:32.091 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:26:32.091 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 20.163660656s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Aug 25 09:26:32.510 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2340, tv_nsec: 48908186 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:26:32.511 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:26:32.512 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:33.45927407Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:35.459351921Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:37.459418151Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:39.459497012Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.452681378Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:41.452709278Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.453626852Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.454261132Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.454375844Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.454610657Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.454860481Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:41.455124815Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:41.459523261Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:43.459584151Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:45.459642051Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:47.459688231Z","hostname":"oxz_switch","pid":2296} Aug 25 09:26:47.474 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:26:47.474 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 35.588060376s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Aug 25 09:26:48.554 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2356, tv_nsec: 135538966 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:26:48.554 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:26:48.555 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:49.459755342Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.45367113Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:26:51.4537002Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.454610414Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.455189133Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.455374006Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.455574709Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.455729541Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:26:51.455935834Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:51.459783142Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:53.459857613Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:55.459905494Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:57.459969454Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:26:59.460031955Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.455053619Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:01.455091289Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.455933012Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.456692733Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.456912297Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.45710681Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.457264502Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:01.457422975Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:01.461011799Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:03.46109419Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:03.516 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:27:03.516 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 51.673826014s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:05.461149761Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:05.938 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2373, tv_nsec: 567297277 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:27:05.939 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:27:05.940 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:07.461178172Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:09.461218412Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.455440435Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:11.455475895Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.456387149Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.456959188Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.457156851Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.457326394Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.457463436Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:11.457593898Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:11.462244988Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:13.46233787Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:15.46239091Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:17.462471Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:27:19.46251438Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:20.901 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:27:20.901 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 69.10589697s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.456717032Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:21.456749523Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.457618196Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.458178234Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.458423648Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.458584011Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.458723933Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:21.458929226Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:24.068 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2391, tv_nsec: 745415974 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:27:24.068 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:27:24.069 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.456330623Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:31.456381094Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.4574169Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.457945757Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.45812585Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.458269342Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.458391804Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:31.458529776Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:39.054 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:27:39.054 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 87.283742513s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.456949731Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:41.456983661Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.457822814Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.458336082Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.458505744Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.458645186Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.458766618Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:41.45890409Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:43.988 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2411, tv_nsec: 682286870 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:27:43.989 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:27:43.990 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.458222983Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:27:51.458274003Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.45908725Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.459641644Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.459879658Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.46001637Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.460147942Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:27:51.460285714Z","hostname":"oxz_switch","pid":2296} Aug 25 09:27:58.992 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:27:58.992 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 107.220933512s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.458178652Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:01.458214632Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.459119966Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.459804776Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.46004961Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.460190902Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.460330754Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:01.460476986Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.458184463Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:11.458215704Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.459066607Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.459605645Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.459788268Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.45992043Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.460087652Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:11.460219084Z","hostname":"oxz_switch","pid":2296} Aug 25 09:28:20.144 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2447, tv_nsec: 837661730 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:28:20.145 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:28:20.146 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.45865349Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:21.45869893Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.459581304Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.460249294Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.460574119Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.460797842Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.461016837Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:21.461225808Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.45929064Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:31.459327631Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.460220334Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.460780233Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.460988876Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.461154698Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.461368191Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:31.461540044Z","hostname":"oxz_switch","pid":2296} Aug 25 09:28:35.147 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:28:35.147 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 143.375664355s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.459779363Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:41.459811124Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.460741198Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.461430748Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.461711332Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.4622391Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.462966801Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:41.463294776Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] neighbor buskin@fe80::e4a1:b6ff:fee4:954a missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-08-25T09:28:49.497019556Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.021936275Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:28:54.021979516Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.05085251Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.06812719Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.068339643Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.068548176Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.0688047Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:28:54.069102964Z","hostname":"oxz_switch","pid":2296} Aug 25 09:29:00.294 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8eaca247-8fb9-4759-acd4-f15a549f2991.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bb191099-ebdd-4b45-8c08-f1e2153936fe.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d2bc0506-35d7-4fbf-857b-4f3f01b8596d.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2487, tv_nsec: 986597930 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 25 09:29:00.295 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 25 09:29:00.296 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.55130135Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:01.551333301Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.5525928Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.5532793Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.553473903Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.553652215Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.553801387Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:01.55395493Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.551067719Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:11.55111248Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.551996113Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.552521101Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.552677894Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.552824826Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.552936917Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:11.553061109Z","hostname":"oxz_switch","pid":2296} Aug 25 09:29:15.298 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Aug 25 09:29:15.300 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 183.528192208s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.55207632Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:21.55211865Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.553343259Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.554050789Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.554217692Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.554372314Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.554500226Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:21.554735869Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.551861874Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:31.551892374Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.552828828Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.553419897Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.5536341Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.553816853Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.553955705Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:31.554148758Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.552307439Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:41.55234176Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.553347725Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.554119827Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.554433221Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.554626074Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.554827177Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:41.5550052Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.552881047Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:29:51.552951008Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.55438082Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.554914968Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.55504729Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.555204732Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.555348694Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:29:51.555627698Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.552920435Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-08-25T09:30:01.552960836Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.553964431Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.5545768Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.554798694Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.554964846Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e4a1:b6ff:fee4:954a on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.55518708Z","hostname":"oxz_switch","pid":2296} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-08-25T09:30:01.555375042Z","hostname":"oxz_switch","pid":2296}