[ Jul 23 10:11:07 Disabled. ] [ Jul 23 10:11:07 Rereading configuration. ] [ Jul 23 10:11:08 Enabled. ] [ Jul 23 10:11:08 Rereading configuration. ] [ Jul 23 10:11:08 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Jul 23 10:11:08 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:08.408894087Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:08.429694624Z","hostname":"oxz_switch","pid":2289} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-07-23T10:11:08.429800486Z","hostname":"oxz_switch","pid":2289} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:08.429811196Z","hostname":"oxz_switch","pid":2289} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:08.430638133Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:09.430065849Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:10.429922484Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:11.429964694Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:12.430058263Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:13.430123703Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:14.430499528Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:15.430475376Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:16.430551395Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:17.430652855Z","hostname":"oxz_switch","pid":2289} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:18.430697474Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] sm initialized with addr fe80::38f8:a2ff:fe92:538f on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.430796384Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2025-07-23T10:11:19.430819255Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] nbr is fe80::f882:2bff:feaf:fc9e@buskin server","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.431325455Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2025-07-23T10:11:19.431403977Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] exchange: listening on [fe80::38f8:a2ff:fe92:538f]:56797","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.43158872Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.431732103Z","hostname":"oxz_switch","pid":2289} [ Jul 23 10:11:19 Rereading configuration. ] [ Jul 23 10:11:19 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 216"). ] + CONTRACT='' + getopts c: o + CONTRACT=216 + getopts c: o + [[ -n 216 ]] + pgrep -c 216 + PARENT_PID=2286 + [[ -n 2286 ]] + pgrep -P 2286 + CHILD_PID=2289 + [[ -n 2289 ]] + kill -USR1 2289 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.797165548Z","hostname":"oxz_switch","pid":2289} [ Jul 23 10:11:19 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:19.801221091Z","hostname":"oxz_switch","pid":2289} Jul 23 10:11:19.802 DEBG registered DTrace probes Jul 23 10:11:19.802 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 23 10:11:19.802 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328 Jul 23 10:11:19.803 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 23 10:11:19.803 DEBG Nexus IP not provided, will use DNS to resolve it Jul 23 10:11:19.803 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/internal-dns/resolver/src/resolver.rs:111 Jul 23 10:11:19.803 DEBG starting producer registration task Jul 23 10:11:19.803 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: a2826c11-418c-4af4-a55c-46138ee2938d, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:279 Jul 23 10:11:19.803 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.016576385Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:11:21.433721991Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.434541967Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2025-07-23T10:11:21.434568808Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::f882:2bff:feaf:fc9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.435404895Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::f882:2bff:feaf:fc9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.436680881Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::f882:2bff:feaf:fc9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.437062329Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::f882:2bff:feaf:fc9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.437231032Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::f882:2bff:feaf:fc9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.437374255Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:11:21.437559669Z","hostname":"oxz_switch","pid":2289} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2025-07-23T10:11:21.43760219Z","hostname":"oxz_switch","pid":2289} Jul 23 10:11:24.805 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 632, tv_nsec: 161598421 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:11:24.806 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:11:24.806 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:11:39.105 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:11:39.105 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 20.004910452s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:11:39.420 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 647, tv_nsec: 478352827 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:11:39.421 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:11:39.421 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:11:54.463 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:11:54.463 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 35.320644726s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:11:54.790 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 662, tv_nsec: 806907925 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:11:54.791 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:11:54.791 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:12:09.834 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:12:09.834 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 50.649871048s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:12:10.802 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 678, tv_nsec: 774774504 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:12:10.802 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:12:10.802 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:12:25.845 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:12:25.845 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 66.617802449s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:12:27.461 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 695, tv_nsec: 388916446 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:12:27.461 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:12:27.462 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:12:42.503 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:12:42.504 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 83.231154875s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:12:44.688 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 712, tv_nsec: 569469145 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:12:44.689 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:12:44.689 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f882:2bff:feaf:fc9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:12:48.930769506Z","hostname":"oxz_switch","pid":2289} Jul 23 10:12:59.708 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:12:59.708 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 100.411855788s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:13:09.891 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 737, tv_nsec: 755767507 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:13:09.892 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:13:09.892 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:13:24.897 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:13:24.897 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 125.59822004s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f882:2bff:feaf:fc9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-23T10:13:36.423879998Z","hostname":"oxz_switch","pid":2289} Jul 23 10:13:37.298 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 765, tv_nsec: 144050277 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:13:37.309 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:13:37.405 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:13:52.417 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 23 10:13:52.420 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 153.120460338s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 Jul 23 10:14:36.416 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("503f98a0-54fd-4258-b2d8-a761939d1032.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("d8901a1f-81d9-4e2e-a334-fe58271d3891.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("ee826bb4-083c-4a45-85c5-33ab74f68864.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 824, tv_nsec: 272065277 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 23 10:14:36.417 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 23 10:14:36.419 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 23 10:14:36.431 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "05b4b44c-fa5c-4e0d-a689-8667c9696672", "content-length": "41", "date": "Wed, 23 Jul 2025 10:14:36 GMT"} }), component: producer-registration-task Jul 23 10:14:36.431 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task {"msg":"withdraw tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:42.675089726Z","hostname":"oxz_switch","pid":2289} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:42.683276294Z","hostname":"oxz_switch","pid":2289} {"msg":"withdraw tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:43.001276795Z","hostname":"oxz_switch","pid":2289} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:43.003425949Z","hostname":"oxz_switch","pid":2289} {"msg":"withdraw tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:43.255984149Z","hostname":"oxz_switch","pid":2289} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd88:4092:f937:3b05::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-23T10:14:43.257765225Z","hostname":"oxz_switch","pid":2289} Jul 23 10:14:51.218 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:1025 Jul 23 10:14:51.226 INFO request completed, latency_us: 8416, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 9e625640-aec6-4d9a-963a-d84daf26d7b4, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:52.219 INFO request completed, latency_us: 650, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 56f1bf23-f7ae-4204-9946-4656eb963d5f, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:53.219 INFO request completed, latency_us: 733, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 0f1eb607-8fb7-433b-abbe-be1593b7425e, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:54.218 INFO request completed, latency_us: 672, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 1e21f71a-c7ed-40a2-93bb-44f3c4bb075f, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:55.219 INFO request completed, latency_us: 716, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 5dada532-c5ae-49a5-9e83-1504abda80e5, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:56.219 INFO request completed, latency_us: 975, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 6cfac77a-c18c-4bab-aefc-9a2f37d6c8d0, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:57.219 INFO request completed, latency_us: 714, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 8cc25aea-e150-475f-98e4-738a8e5f69a1, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:58.219 INFO request completed, latency_us: 657, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 76323b19-f2ea-4372-b5d8-d7a4387a4ddc, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:14:59.219 INFO request completed, latency_us: 608, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 70f355d1-d002-4e89-b33c-70aa2494bee5, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 Jul 23 10:15:00.219 INFO request completed, latency_us: 614, response_code: 200, uri: /a2826c11-418c-4af4-a55c-46138ee2938d, method: GET, req_id: 80325335-ed67-4461-85b0-dd9cf2ce8c73, remote_addr: [fd00:1122:3344:101::d]:42247, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867