[ Nov 8 10:27:09 Disabled. ] [ Nov 8 10:27:09 Rereading configuration. ] [ Nov 8 10:27:10 Enabled. ] [ Nov 8 10:27:10 Rereading configuration. ] [ Nov 8 10:27:10 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Nov 8 10:27:10 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:10.42797162Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:10.449921417Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-11-08T10:27:10.449971578Z","hostname":"oxz_switch","pid":2302} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:10.449979128Z","hostname":"oxz_switch","pid":2302} {"msg":"admin: listening on [::]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:10.451307061Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:11.449999408Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:12.45011217Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:13.450192141Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:14.450263852Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:15.450356223Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:16.450427644Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:17.450529245Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:18.450604615Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:19.450679107Z","hostname":"oxz_switch","pid":2302} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:20.450758578Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] sm initialized with addr fe80::ace7:99ff:fe71:f574 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:21.450899629Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-11-08T10:27:21.451061991Z","hostname":"oxz_switch","pid":2302} [ Nov 8 10:27:22 Rereading configuration. ] [ Nov 8 10:27:22 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2300 + [[ -n 2300 ]] + pgrep -P 2300 + CHILD_PID=2302 + [[ -n 2302 ]] + kill -USR1 2302 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:22.257330288Z","hostname":"oxz_switch","pid":2302} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:22.259325869Z","hostname":"oxz_switch","pid":2302} Nov 08 10:27:22.261 DEBG registered DTrace probes [ Nov 8 10:27:22 Method "refresh" exited with status 0. ] Nov 08 10:27:22.262 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Nov 08 10:27:22.262 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Nov 08 10:27:22.262 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Nov 08 10:27:22.262 DEBG Nexus IP not provided, will use DNS to resolve it Nov 08 10:27:22.263 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Nov 08 10:27:22.264 DEBG starting producer registration task Nov 08 10:27:22.264 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: dac62a7d-791f-432a-8ce1-900a5d3422f6, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Nov 08 10:27:22.264 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task Nov 08 10:27:22.266 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41461, tv_nsec: 122837521 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:27:22.267 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:27:22.270 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:22.974012289Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] nbr is fe80::4022:b7ff:fe8a:3582@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.451694828Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-11-08T10:27:23.45180715Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] exchange: listening on [fe80::ace7:99ff:fe71:f574]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.452030972Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.452215534Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.454551348Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-11-08T10:27:23.454560938Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::4022:b7ff:fe8a:3582 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.45570381Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::4022:b7ff:fe8a:3582 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.456848992Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::4022:b7ff:fe8a:3582 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.457147435Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::4022:b7ff:fe8a:3582 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.457360307Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::4022:b7ff:fe8a:3582 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.457569049Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.457895003Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-11-08T10:27:23.45859751Z","hostname":"oxz_switch","pid":2302} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd6d:2cc1:6dd7:81a7::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:23.980288584Z","hostname":"oxz_switch","pid":2302} Nov 08 10:27:37.449 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:27:37.449 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 15.008124496s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 08 10:27:37.789 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41476, tv_nsec: 469558618 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:27:37.790 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:27:37.790 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:39.628136679Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:27:41.628206107Z","hostname":"oxz_switch","pid":2302} Nov 08 10:27:52.751 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:27:52.751 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 30.352158158s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 08 10:27:53.097 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41491, tv_nsec: 819441961 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:27:53.098 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:27:53.098 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:27:53.109745215Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:05.629433361Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:07.629520299Z","hostname":"oxz_switch","pid":2302} Nov 08 10:28:08.059 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:28:08.059 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 45.701405324s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 08 10:28:08.940 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41507, tv_nsec: 705183484 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:28:08.941 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:28:08.941 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:09.629525386Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:11.629561403Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:13.62959103Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:15.629636448Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:17.629643464Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:19.62966758Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:21.629755658Z","hostname":"oxz_switch","pid":2302} {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:28:23.031217688Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:23.629757094Z","hostname":"oxz_switch","pid":2302} Nov 08 10:28:23.901 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:28:23.902 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 61.587064246s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Nov 08 10:28:25.471 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41524, tv_nsec: 281557678 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:28:25.472 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:28:25.473 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:25.629784161Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:27.629824208Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:29.629872654Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:31.62990584Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:33.629965627Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:35.630021405Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:37.630030131Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:39.630079008Z","hostname":"oxz_switch","pid":2302} Nov 08 10:28:40.434 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:28:40.434 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 78.16455472s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:41.630131075Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:43.630175431Z","hostname":"oxz_switch","pid":2302} Nov 08 10:28:43.934 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41542, tv_nsec: 793852988 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:28:43.935 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:28:43.935 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:45.630223638Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:47.630248946Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:49.630286393Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:51.6303543Z","hostname":"oxz_switch","pid":2302} {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:28:52.95174811Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:53.630387517Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:55.630451875Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:57.630502182Z","hostname":"oxz_switch","pid":2302} Nov 08 10:28:58.896 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:28:58.896 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 96.676578919s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:28:59.630574238Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:01.630612115Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:03.630633792Z","hostname":"oxz_switch","pid":2302} Nov 08 10:29:03.976 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41562, tv_nsec: 890670435 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:29:03.977 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:29:03.978 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:05.630685069Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:07.630727355Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:09.630769733Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:11.630814749Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:13.630857326Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:15.630895753Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:17.63097519Z","hostname":"oxz_switch","pid":2302} Nov 08 10:29:18.939 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:29:18.939 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 116.773612007s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:19.630985537Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:21.631036324Z","hostname":"oxz_switch","pid":2302} {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:29:22.873330804Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:23.631104301Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:25.631137317Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:27.631158794Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:29.631213301Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:31.632406279Z","hostname":"oxz_switch","pid":2302} Nov 08 10:29:32.417 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("066b912f-35f9-4a2a-9a9b-f8f67b74da6c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("291cdda7-c59b-4bbb-91e4-57cd2fc467ae.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e399b87b-d5aa-4e17-83d2-15ff97eba4ab.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 41591, tv_nsec: 406639296 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Nov 08 10:29:32.420 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Nov 08 10:29:32.427 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:35.631598575Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:37.631617281Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:39.632635049Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:41.632686985Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:43.633710123Z","hostname":"oxz_switch","pid":2302} {"msg":"[tfportrear0_0] neighbor sock@fe80::4022:b7ff:fe8a:3582 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-11-08T10:29:45.63473893Z","hostname":"oxz_switch","pid":2302} Nov 08 10:29:47.388 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Nov 08 10:29:47.388 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 145.296542632s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"peer change request","v":0,"name":"slog-rs","level":30,"time":"2026-11-08T10:29:52.814306806Z","hostname":"oxz_switch","pid":2302,"current":"{}","to_remove":"Difference(Iterate(Iter([])))","to_create":"Difference(Iterate(Iter([])))","requested":"{}"}