[ Jan  5 22:27:14 Disabled. ]
[ Jan  5 22:27:14 Rereading configuration. ]
[ Jan  5 22:27:15 Enabled. ]
[ Jan  5 22:27:15 Rereading configuration. ]
[ Jan  5 22:27:15 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ Jan  5 22:27:15 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:15.304159865Z","hostname":"oxz_switch","pid":2294}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-01-05T22:27:15.324226471Z","hostname":"oxz_switch","pid":2294}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:15.324247212Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:15.324257172Z","hostname":"oxz_switch","pid":2294}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:15.325379821Z","hostname":"oxz_switch","pid":2294}
{"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:15.32593235Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:16.324415611Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:17.32454597Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:18.324615226Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:19.324703393Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:20.324848153Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:21.324909529Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:22.324987155Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:23.325079662Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:24.325221762Z","hostname":"oxz_switch","pid":2294}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:25.325300908Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::24f8:78ff:fe78:55d4 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.325418596Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-01-05T22:27:26.325504799Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] nbr is fe80::a2:2ff:fe8f:3dc1@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.326142591Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-01-05T22:27:26.326224584Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::24f8:78ff:fe78:55d4]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.326434151Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.326626098Z","hostname":"oxz_switch","pid":2294}
[ Jan  5 22:27:26 Rereading configuration. ]
[ Jan  5 22:27:26 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 216"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=216
+ getopts c: o
+ [[ -n 216 ]]
+ pgrep -c 216
+ PARENT_PID=2293
+ [[ -n 2293 ]]
+ pgrep -P 2293
+ CHILD_PID=2294
+ [[ -n 2294 ]]
+ kill -USR1 2294
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.735028185Z","hostname":"oxz_switch","pid":2294}
[ Jan  5 22:27:26 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:26.737831192Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:27:26.738 DEBG registered DTrace probes
Jan 05 22:27:26.739 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jan 05 22:27:26.739 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.5/src/server.rs:328
Jan 05 22:27:26.739 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jan 05 22:27:26.739 DEBG Nexus IP not provided, will use DNS to resolve it
Jan 05 22:27:26.739 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/internal-dns/resolver/src/resolver.rs:144
Jan 05 22:27:26.739 DEBG starting producer registration task
Jan 05 22:27:26.739 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 69d88630-636e-4897-b5f5-e77723054463, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:279
Jan 05 22:27:26.739 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Net {\n                addr: 0.0.0.0,\n                width: 0,\n            },\n        ),\n        boundary_addr: fd76:b985:eb4e:88::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:27.930407692Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:28.330555692Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.331656901Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-01-05T22:27:28.331681551Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::a2:2ff:fe8f:3dc1 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.33365665Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::a2:2ff:fe8f:3dc1 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.335070419Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::a2:2ff:fe8f:3dc1 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.33539734Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::a2:2ff:fe8f:3dc1 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.335647589Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::a2:2ff:fe8f:3dc1 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.335912178Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-01-05T22:27:28.336216749Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-01-05T22:27:28.336230319Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:27:31.743 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 664, tv_nsec: 508310556 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:27:31.744 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:27:31.744 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:45.03502912Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:47.035072238Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:27:47.444 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:27:47.444 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 20.005271483s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
Jan 05 22:27:47.801 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 679, tv_nsec: 867336741 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:27:47.801 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:27:47.801 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:49.035115836Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:51.035164396Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:53.036176775Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:55.036236858Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:57.037034981Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:27:59.037085555Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:01.037163373Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:02.761 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:28:02.761 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 35.364468724s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:03.038163405Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:03.426 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 695, tv_nsec: 536163616 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:28:03.427 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:28:03.427 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:05.038202514Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:07.038236932Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:09.038322452Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:11.039312354Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:13.039330471Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:15.039377019Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:17.040366829Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:18.388 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:28:18.388 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 51.033983802s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:19.040447349Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:19.821 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 711, tv_nsec: 976070298 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:28:19.822 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:28:19.822 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:21.040481948Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:23.040508606Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:25.040577385Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:27.041594857Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:29.041623086Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:31.042658528Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:33.042701074Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:34.782 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:28:34.782 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 67.472778366s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:35.042766295Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:36.106 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 728, tv_nsec: 304570320 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:28:36.106 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:28:36.106 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:37.042801913Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:39.043822295Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:41.044858888Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:43.044897586Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:45.044970386Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:47.045017084Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:49.045058782Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:51.0450876Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:51.067 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:28:51.067 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 83.802017943s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:53.045154099Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:28:53.205 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 745, tv_nsec: 450953662 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:28:53.206 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:28:53.206 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:55.045212068Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:57.045262867Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:28:59.045302334Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:01.045353053Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:03.046379116Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:05.046474155Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:07.046510233Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:29:08.166 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:29:08.166 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 100.947497294s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:09.046557271Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:11.04661873Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:13.047035251Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:15.04708625Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:17.048122803Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:19.049144275Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:29:19.525 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 771, tv_nsec: 842623863 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:29:19.526 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:29:19.526 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:21.049174952Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:23.049248132Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:25.049262899Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:27.049330518Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:29.049366534Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:31.049548877Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:33.049627937Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:29:34.486 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 05 22:29:34.486 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 127.339363321s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/b36f032/oximeter/producer/src/lib.rs:371
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:35.050625458Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:37.050708918Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:43.051798646Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:45.052110484Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:47.051883093Z","hostname":"oxz_switch","pid":2294}
Jan 05 22:29:47.827 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("77861647-cebf-4e85-bdde-48b1a190f0ce.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("98249b9a-2f6e-46a4-be1e-673a1bba11fa.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("a85b9783-de7f-4744-8cf3-ac8a31b0cd75.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 800, tv_nsec: 221598572 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 05 22:29:47.828 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jan 05 22:29:47.828 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:49.051922461Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:51.051956379Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:53.052012128Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:55.052029186Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:57.052088625Z","hostname":"oxz_switch","pid":2294}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::a2:2ff:fe8f:3dc1 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-01-05T22:29:59.052105772Z","hostname":"oxz_switch","pid":2294}