[ Jul 18 19:12:44 Disabled. ]
[ Jul 18 19:12:44 Rereading configuration. ]
[ Jul 18 19:12:45 Enabled. ]
[ Jul 18 19:12:45 Rereading configuration. ]
[ Jul 18 19:12:45 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ Jul 18 19:12:45 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:45.135738818Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-07-18T19:12:45.162056382Z","hostname":"oxz_switch","pid":2283}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:45.162093603Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:45.163517805Z","hostname":"oxz_switch","pid":2283}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:45.164059663Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:46.163675162Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:47.163795259Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:48.163873535Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:49.163963481Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:50.164060308Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:51.164199075Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:52.164259341Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:53.164329317Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:54.164401543Z","hostname":"oxz_switch","pid":2283}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:55.164467499Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::cc79:e4ff:feae:b59b on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.164559026Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:12:56.164595686Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] nbr is fe80::e02c:edff:fe35:1da8@buskin server","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.165132514Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:12:56.165195825Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::cc79:e4ff:feae:b59b]:56797","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.165353868Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.1654744Z","hostname":"oxz_switch","pid":2283}
[ Jul 18 19:12:56 Rereading configuration. ]
[ Jul 18 19:12:56 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 215"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=215
+ getopts c: o
+ [[ -n 215 ]]
+ pgrep -c 215
+ PARENT_PID=2280
+ [[ -n 2280 ]]
+ pgrep -P 2280
+ CHILD_PID=2283
+ [[ -n 2283 ]]
+ kill -USR1 2283
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.501702729Z","hostname":"oxz_switch","pid":2283}
[ Jul 18 19:12:56 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:56.503457286Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:12:56.504 DEBG registered DTrace probes
Jul 18 19:12:56.504 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jul 18 19:12:56.504 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jul 18 19:12:56.504 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jul 18 19:12:56.505 DEBG Nexus IP not provided, will use DNS to resolve it
Jul 18 19:12:56.505 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/internal-dns/resolver/src/resolver.rs:111
Jul 18 19:12:56.505 DEBG starting producer registration task
Jul 18 19:12:56.505 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 4f77390a-9897-4917-96cd-2615b81a6968, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:279
Jul 18 19:12:56.505 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Net {\n                addr: 0.0.0.0,\n                width: 0,\n            },\n        ),\n        boundary_addr: fd62:890d:d43f:3afc::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:57.693674886Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:12:58.167376279Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.168491026Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:12:58.168513577Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.169309419Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.170648479Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.170903963Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.171132817Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.17131183Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:12:58.171470152Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:12:58.171493583Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:13:01.508 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 801, tv_nsec: 366981645 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:13:01.508 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:13:01.508 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jul 18 19:13:16.344 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:13:16.344 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 20.005092409s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
Jul 18 19:13:16.651 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 816, tv_nsec: 677129859 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:13:16.651 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:13:16.652 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:18.000326569Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:22.001393072Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:24.002395149Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:26.00340925Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:28.004429609Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:30.005445417Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:13:31.612 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:13:31.612 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 35.314963668s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
Jul 18 19:13:31.931 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 831, tv_nsec: 998967822 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:13:31.932 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:13:31.932 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:32.006451275Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:34.007493203Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 expire","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:40.086903987Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] expiring peer fe80::e02c:edff:fe35:1da8 due to discovery event","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:40.089572768Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] nbr is fe80::e02c:edff:fe35:1da8@buskin server","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.089586788Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] removing routes 5 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.089608028Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] failed to remove tunnel routes: {} OPTE driver is not attached","v":0,"name":"slog-rs","level":50,"time":"2025-07-18T19:13:40.190393512Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] redistributing expire to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:13:40.190427943Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:13:40.190453693Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::cc79:e4ff:feae:b59b]:56797","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.191070727Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.192462644Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.22013167Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:13:40.220158831Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.223667215Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.224793272Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.225076417Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.22531645Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::e02c:edff:fe35:1da8 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.225556324Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2025-07-18T19:13:40.225780288Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2025-07-18T19:13:40.225796978Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:45.986327305Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:13:46.895 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:13:46.897 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 50.640810023s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
Jul 18 19:13:47.483 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 847, tv_nsec: 592677445 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:13:47.484 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:13:47.487 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:47.987318604Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:49.988342682Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:51.989354551Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:53.99036995Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:55.991590972Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:57.992607971Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:13:59.993627541Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:02.447 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:14:02.447 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 66.233208156s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:03.994911527Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:04.163 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 864, tv_nsec: 318395043 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:14:04.164 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:14:04.164 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:05.995931105Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:07.996942984Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:09.997961273Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:11.998976772Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:13.999992581Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:16.00101168Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:18.002062429Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:19.124 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:14:19.124 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 82.955740205s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:20.003066468Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:22.004082476Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:22.588 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 882, tv_nsec: 793069651 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:14:22.589 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:14:22.589 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:24.005099205Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:26.006112564Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:28.007129302Z","hostname":"oxz_switch","pid":2283}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:30.008143601Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:37.550 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 18 19:14:37.550 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 101.430816829s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::e02c:edff:fe35:1da8 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2025-07-18T19:14:43.070429857Z","hostname":"oxz_switch","pid":2283}
Jul 18 19:14:49.254 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("33fb85a6-524f-4ef3-b2bc-002d659a2026.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4627777a-3582-477b-a211-43f686d83b54.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("8d963aca-f065-4d8f-a1ae-e4fe8d4ec172.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 909, tv_nsec: 525414571 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 18 19:14:49.256 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jul 18 19:14:49.301 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task