[ Apr 24 17:27:26 Disabled. ] [ Apr 24 17:27:26 Rereading configuration. ] [ Apr 24 17:27:27 Enabled. ] [ Apr 24 17:27:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Apr 24 17:27:27 Method "start" exited with status 0. ] [ Apr 24 17:27:27 Rereading configuration. ] [ Apr 24 17:27:27 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 212"). ] + CONTRACT='' + getopts c: o + CONTRACT=212 + getopts c: o + [[ -n 212 ]] + pgrep -c 212 + PARENT_PID=2285 + [[ -n 2285 ]] + pgrep -P 2285 + CHILD_PID=2286 + [[ -n 2286 ]] + kill -USR1 2286 [ Apr 24 17:27:27 Method "refresh" exited with status 0. ] [ Apr 24 17:27:27 Stopping because all processes in service exited. ] [ Apr 24 17:27:27 Executing stop method (:kill). ] [ Apr 24 17:27:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Apr 24 17:27:27 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:27.17695405Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:27.196831549Z","hostname":"oxz_switch","pid":2308} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-04-24T17:27:27.19689859Z","hostname":"oxz_switch","pid":2308} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:27.196943932Z","hostname":"oxz_switch","pid":2308} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:27.197742983Z","hostname":"oxz_switch","pid":2308} {"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:27.198254256Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:28.197009879Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:29.197136179Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:30.197202566Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:31.197294915Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:32.197378483Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:33.197449231Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:34.19756047Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:35.197658008Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:36.197743256Z","hostname":"oxz_switch","pid":2308} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:37.197826305Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] sm initialized with addr fe80::4412:4fff:fe1e:84e6 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:38.197943364Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-04-24T17:27:38.197981895Z","hostname":"oxz_switch","pid":2308} [ Apr 24 17:27:38 Rereading configuration. ] [ Apr 24 17:27:38 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 217"). ] + CONTRACT='' + getopts c: o + CONTRACT=217 + getopts c: o + [[ -n 217 ]] + pgrep -c 217 + PARENT_PID=2307 + [[ -n 2307 ]] + pgrep -P 2307 + CHILD_PID=2308 + [[ -n 2308 ]] + kill -USR1 2308 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:38.703891807Z","hostname":"oxz_switch","pid":2308} [ Apr 24 17:27:38 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:38.706986068Z","hostname":"oxz_switch","pid":2308} Apr 24 17:27:38.707 DEBG registered DTrace probes Apr 24 17:27:38.708 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Apr 24 17:27:38.708 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328 Apr 24 17:27:38.708 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Apr 24 17:27:38.708 DEBG Nexus IP not provided, will use DNS to resolve it Apr 24 17:27:38.708 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/internal-dns/resolver/src/resolver.rs:144 Apr 24 17:27:38.708 DEBG starting producer registration task Apr 24 17:27:38.708 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:283 Apr 24 17:27:38.709 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fd1e:9bc1:c3f:6c76::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:39.912481322Z","hostname":"oxz_switch","pid":2308} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd1e:9bc1:c3f:6c76::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:39.912680147Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fd1e:9bc1:c3f:6c76::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:39.912762169Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] nbr is fe80::18f9:6ff:fe8a:edc6@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.198373297Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-04-24T17:27:40.198441209Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] exchange: listening on [fe80::4412:4fff:fe1e:84e6]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.198631274Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.198765847Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.200958974Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-04-24T17:27:40.200971104Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::18f9:6ff:fe8a:edc6 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.201823727Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::18f9:6ff:fe8a:edc6 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.202937136Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::18f9:6ff:fe8a:edc6 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.203202183Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::18f9:6ff:fe8a:edc6 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.203402258Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::18f9:6ff:fe8a:edc6 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.203723846Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-04-24T17:27:40.203926402Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-04-24T17:27:40.203951862Z","hostname":"oxz_switch","pid":2308} Apr 24 17:27:43.711 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 671, tv_nsec: 131372008 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:27:43.712 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:27:43.714 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:27:58.470413848Z","hostname":"oxz_switch","pid":2308} Apr 24 17:27:58.982 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:27:58.983 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 20.007273989s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:27:59.311 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 686, tv_nsec: 466216379 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:27:59.312 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:27:59.313 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:00.471574722Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:02.471674882Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:04.472685008Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:06.473729079Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:08.474728147Z","hostname":"oxz_switch","pid":2308} {"msg":"[tfportrear0_0] neighbor buskin@fe80::18f9:6ff:fe8a:edc6 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-04-24T17:28:11.586453646Z","hostname":"oxz_switch","pid":2308} Apr 24 17:28:14.285 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:28:14.287 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 35.351041153s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:28:14.587 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 701, tv_nsec: 783029920 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:28:14.590 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:28:14.598 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:28:29.559 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:28:29.559 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 50.667224288s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:28:30.609 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 717, tv_nsec: 848688299 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:28:30.609 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:28:30.610 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:28:45.611 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:28:45.611 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 66.723327027s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:28:46.951 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 734, tv_nsec: 175146287 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:28:46.955 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:28:47.000 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:29:02.005 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:29:02.008 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 83.12051893s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:29:05.850 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 753, tv_nsec: 91254354 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:29:05.851 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:29:05.854 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:29:20.855 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 24 17:29:20.855 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out, delay: 101.967568956s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:29:27.689 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 774, tv_nsec: 930952285 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:29:27.690 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:29:27.691 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:29:27.701 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task Apr 24 17:29:27.702 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): client error (Connect): tcp connect error: Connection refused (os error 146), delay: 108.814199378s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379 Apr 24 17:29:38.775 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46647e07-ece7-4ab7-9ddb-14bc67b060bf.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("7c0f82d0-9104-456b-8d63-d9f25c4db110.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("d97212d8-4e64-4404-bea9-5b642fe25e69.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 786, tv_nsec: 17224691 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 24 17:29:38.776 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 24 17:29:38.780 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 24 17:29:38.797 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "73f0e7fe-f94d-4f54-b86b-0c356e81c144", "content-length": "41", "date": "Fri, 24 Apr 2026 17:29:38 GMT"} }), component: producer-registration-task Apr 24 17:29:38.798 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task Apr 24 17:29:50.479 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:1025 Apr 24 17:29:50.490 INFO request completed, latency_us: 10259, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 8b8070e6-fd70-46ea-b626-87d9172ca3ff, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:51.480 INFO request completed, latency_us: 600, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 25d16ec2-6a0b-48fe-a999-360d29efae44, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:52.480 INFO request completed, latency_us: 681, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 154cc6f7-cfca-4ade-bfdd-47fbf9daf299, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:53.480 INFO request completed, latency_us: 728, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 946a047d-2f6b-4866-a59c-f7538de5500f, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:54.480 INFO request completed, latency_us: 691, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: ca040669-685d-412f-8736-4d8fb0475be3, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:55.480 INFO request completed, latency_us: 604, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: cea42620-33a2-46d3-b58a-81fcb3abeaeb, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:56.480 INFO request completed, latency_us: 690, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: f42036b1-5226-4c1b-a232-7490ffc6e3c7, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:57.480 INFO request completed, latency_us: 700, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: f546d013-5b70-4aa3-bff0-2d328bc7664c, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:58.480 INFO request completed, latency_us: 786, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: fdc808d5-f4b8-445d-af59-9c3aa5af5e38, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:29:59.480 INFO request completed, latency_us: 663, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 40b0fc05-bc2a-48df-9dfc-340c0fad2642, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:30:00.480 INFO request completed, latency_us: 607, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: a5c99254-4746-4e09-9d66-14949c0d62d3, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867 Apr 24 17:30:01.480 INFO request completed, latency_us: 700, response_code: 200, uri: /9cd3ef8c-8d55-4f2f-b226-99bd4bcb0f72, method: GET, req_id: 562750b9-158b-45f0-b00c-d827b12fd435, remote_addr: [fd00:1122:3344:101::d]:36140, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867