[ Jul 2 06:12:55 Disabled. ] [ Jul 2 06:12:55 Rereading configuration. ] [ Jul 2 06:12:55 Enabled. ] [ Jul 2 06:12:55 Rereading configuration. ] [ Jul 2 06:12:55 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Jul 2 06:12:55 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:12:55.672354936Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:12:55.69365954Z","hostname":"oxz_switch","pid":2290} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-07-02T06:12:55.693681351Z","hostname":"oxz_switch","pid":2290} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:12:55.693689081Z","hostname":"oxz_switch","pid":2290} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:12:55.694541657Z","hostname":"oxz_switch","pid":2290} {"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:12:55.695032086Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:12:56.69375336Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:12:57.69385733Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:12:58.6939656Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:12:59.694080251Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:00.698059205Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:01.696235448Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:02.696311598Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:03.696414348Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:04.696545628Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:05.696610487Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sm initialized with addr fe80::4c44:5cff:fe14:44a3 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:06.696745268Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:13:06.696780708Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] nbr is fe80::5848:fcff:fe62:ec9e@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:06.69738673Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:13:06.69740305Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] exchange: listening on [fe80::4c44:5cff:fe14:44a3]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:06.697582734Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:06.697681045Z","hostname":"oxz_switch","pid":2290} [ Jul 2 06:13:07 Rereading configuration. ] [ Jul 2 06:13:07 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2289 + [[ -n 2289 ]] + pgrep -P 2289 + CHILD_PID=2290 + [[ -n 2290 ]] + kill -USR1 2290 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:07.14132986Z","hostname":"oxz_switch","pid":2290} [ Jul 2 06:13:07 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:07.14498556Z","hostname":"oxz_switch","pid":2290} Jul 02 06:13:07.145 DEBG registered DTrace probes Jul 02 06:13:07.146 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 02 06:13:07.146 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335 Jul 02 06:13:07.146 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Jul 02 06:13:07.146 DEBG Nexus IP not provided, will use DNS to resolve it Jul 02 06:13:07.146 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144 Jul 02 06:13:07.146 DEBG starting producer registration task Jul 02 06:13:07.147 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: e16cd9c3-a740-480a-8062-e0979d5d86f5, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280 Jul 02 06:13:07.147 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task Jul 02 06:13:07.148 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 1.542869ms, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:13:07.487 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 340.378226ms, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:13:07.887 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 688, tv_nsec: 959153220 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:13:07.887 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:13:07.889 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fdac:73d7:6d1f:5dfa::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.327220364Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:08.699372474Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.700379193Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:13:08.700406973Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.701688628Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.703036773Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.703370149Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.703643165Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.70393772Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:13:08.704235706Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:13:08.705164614Z","hostname":"oxz_switch","pid":2290} Jul 02 06:13:22.697 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 02 06:13:22.697 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 15.743978463s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:13:24.097 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 705, tv_nsec: 367067525 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:13:24.098 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:13:24.098 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:24.504972732Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:26.505001283Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:28.505038712Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:30.50506164Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:32.505688879Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:34.518076865Z","hostname":"oxz_switch","pid":2290} Jul 02 06:13:39.059 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 02 06:13:39.059 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 32.150650617s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:13:40.946 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 722, tv_nsec: 257910785 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:13:40.946 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:13:40.947 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:13:44.559809707Z","hostname":"oxz_switch","pid":2290} Jul 02 06:13:55.949 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 02 06:13:55.949 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 49.041736511s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:14:00.652 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 741, tv_nsec: 963794639 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:14:00.653 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:14:00.655 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:14:05.051097937Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::5848:fcff:fe62:ec9e expire","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:14:15.124085953Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] expiring peer fe80::5848:fcff:fe62:ec9e due to discovery event","v":0,"name":"slog-rs","level":40,"time":"2026-07-02T06:14:15.150941902Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 5 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:15.157083379Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] nbr is fe80::5848:fcff:fe62:ec9e@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:15.177223141Z","hostname":"oxz_switch","pid":2290} Jul 02 06:14:15.681 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 02 06:14:15.760 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 68.773876384s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 {"msg":"[tfportrear0_0] failed to remove tunnel routes: {} OPTE driver is not attached","v":0,"name":"slog-rs","level":50,"time":"2026-07-02T06:14:15.855511666Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing expire to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:14:15.855575088Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:14:15.885296771Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] exchange: listening on [fe80::4c44:5cff:fe14:44a3]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:15.906350531Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:15.947174095Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:16.056205083Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:14:16.056236824Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:16.064449119Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:17.845964182Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:17.850344245Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:17.853506045Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::5848:fcff:fe62:ec9e on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:17.853905602Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-02T06:14:17.855623555Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-07-02T06:14:17.856768167Z","hostname":"oxz_switch","pid":2290} Jul 02 06:14:20.645 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 761, tv_nsec: 955838303 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:14:20.647 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:14:20.650 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 02 06:14:35.653 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Jul 02 06:14:35.653 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 88.7449397s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376 Jul 02 06:14:56.222 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1932ef4c-8532-4db6-9e2a-a9c76a32e899.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6a7f7c01-d53d-4c32-b764-70db9a6b0544.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("e354872d-d25b-4ea4-b14d-12c9be4ccd30.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 797, tv_nsec: 533287470 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Jul 02 06:14:56.223 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task Jul 02 06:14:56.224 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task Jul 02 06:14:56.290 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "16ba7cc9-93a7-47f7-929e-f93f8beb3722", "content-length": "41", "date": "Thu, 02 Jul 2026 06:14:56 GMT"} }), component: producer-registration-task Jul 02 06:14:56.291 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task Jul 02 06:14:56.342 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:1057 Jul 02 06:14:56.355 INFO request completed, latency_us: 12175, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: 6e368f55-949a-421a-9c96-377de64bf136, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874 Jul 02 06:14:57.336 INFO request completed, latency_us: 602, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: 2018e019-0c97-4dd5-9fc8-26204e151624, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874 Jul 02 06:14:58.336 INFO request completed, latency_us: 601, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: b0e49cc4-4950-46ed-93ac-990c85c89d91, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874 Jul 02 06:14:59.336 INFO request completed, latency_us: 637, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: f30c8fc5-9509-46ca-8afb-7fb7e5d9a56c, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874 Jul 02 06:15:00.336 INFO request completed, latency_us: 613, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: d0c29d9b-788b-4651-b4ab-07e71c664a4e, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874 Jul 02 06:15:01.336 INFO request completed, latency_us: 721, response_code: 200, uri: /e16cd9c3-a740-480a-8062-e0979d5d86f5, method: GET, req_id: f798f1be-7bb5-4941-b2b8-4df3abf11bb5, remote_addr: [fd00:1122:3344:101::d]:51927, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874