[ May 26 23:08:10 Disabled. ]
[ May 26 23:08:10 Rereading configuration. ]
[ May 26 23:08:11 Enabled. ]
[ May 26 23:08:11 Rereading configuration. ]
[ May 26 23:08:11 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ May 26 23:08:11 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:11.624310653Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:11.644087695Z","hostname":"oxz_switch","pid":2290}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-05-26T23:08:11.644112626Z","hostname":"oxz_switch","pid":2290}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:11.644121876Z","hostname":"oxz_switch","pid":2290}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:11.644887794Z","hostname":"oxz_switch","pid":2290}
{"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:11.645403347Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:12.644251153Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:13.644357791Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:14.644400306Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:15.644617367Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:16.644686913Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:17.644761639Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:18.644892537Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:19.644955804Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:20.645069462Z","hostname":"oxz_switch","pid":2290}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:21.645124768Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::9870:4fff:feb2:9393 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:22.645250275Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-05-26T23:08:22.645280656Z","hostname":"oxz_switch","pid":2290}
[ May 26 23:08:23 Rereading configuration. ]
[ May 26 23:08:23 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=213
+ getopts c: o
+ [[ -n 213 ]]
+ pgrep -c 213
+ PARENT_PID=2287
+ [[ -n 2287 ]]
+ pgrep -P 2287
+ CHILD_PID=2290
+ [[ -n 2290 ]]
+ kill -USR1 2290
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:23.314072956Z","hostname":"oxz_switch","pid":2290}
[ May 26 23:08:23 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:23.317072848Z","hostname":"oxz_switch","pid":2290}
May 26 23:08:23.318 DEBG registered DTrace probes
May 26 23:08:23.319 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
May 26 23:08:23.319 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328
May 26 23:08:23.319 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
May 26 23:08:23.319 DEBG Nexus IP not provided, will use DNS to resolve it
May 26 23:08:23.319 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/internal-dns/resolver/src/resolver.rs:144
May 26 23:08:23.319 DEBG starting producer registration task
May 26 23:08:23.320 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 5ff07680-a497-4f4f-b278-8c9955051423, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:283
May 26 23:08:23.320 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fda5:cf52:fa98:28b3::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:24.534259146Z","hostname":"oxz_switch","pid":2290}
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Net {\n                addr: 0.0.0.0,\n                width: 0,\n            },\n        ),\n        boundary_addr: fda5:cf52:fa98:28b3::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.534479801Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fda5:cf52:fa98:28b3::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:24.534559043Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] nbr is fe80::9025:5bff:fe00:1b53@buskin server","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.645749687Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-05-26T23:08:24.645768447Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::9870:4fff:feb2:9393]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.645955692Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.646084105Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.648219856Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-05-26T23:08:24.648232266Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::9025:5bff:fe00:1b53 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.648948403Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::9025:5bff:fe00:1b53 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.650122631Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::9025:5bff:fe00:1b53 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.650394778Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::9025:5bff:fe00:1b53 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.650665474Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::9025:5bff:fe00:1b53 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.650836468Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-05-26T23:08:24.651040673Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-05-26T23:08:24.651048883Z","hostname":"oxz_switch","pid":2290}
May 26 23:08:28.323 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 838, tv_nsec: 463223891 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:08:28.323 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:08:28.326 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:43.90829691Z","hostname":"oxz_switch","pid":2290}
May 26 23:08:44.582 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:08:44.582 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 20.007207889s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:08:44.729 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 853, tv_nsec: 614224996 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:08:44.730 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:08:44.731 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:45.909330814Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:47.909367204Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:49.909436485Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:51.909472134Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:53.909527435Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:08:55.909574725Z","hostname":"oxz_switch","pid":2290}
May 26 23:08:59.691 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:08:59.691 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 35.157410364s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:08:59.980 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 868, tv_nsec: 906313387 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:08:59.980 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:08:59.981 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:07.911921415Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:09.912912228Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:11.912977035Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:13.913980984Z","hostname":"oxz_switch","pid":2290}
May 26 23:09:14.942 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:09:14.942 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 50.449846854s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:09:15.847 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 884, tv_nsec: 817445262 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:09:15.847 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:09:15.848 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:15.914057532Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:17.914098958Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:19.914166382Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:21.914203301Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:23.914257099Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:25.914332875Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:27.915324434Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:29.916390471Z","hostname":"oxz_switch","pid":2290}
May 26 23:09:30.808 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:09:30.808 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 66.359961915s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:31.916454833Z","hostname":"oxz_switch","pid":2290}
May 26 23:09:32.470 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 901, tv_nsec: 486529866 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:09:32.471 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:09:32.472 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:33.916493725Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:35.916539202Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:37.916568624Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:39.917588989Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:09:41.917643178Z","hostname":"oxz_switch","pid":2290}
May 26 23:09:47.432 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:09:47.432 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 83.029124459s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:09:52.025 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 921, tv_nsec: 94126115 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:09:52.026 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:09:52.027 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
May 26 23:10:06.987 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:10:06.987 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 102.637172851s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:10:12.032 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 941, tv_nsec: 155763190 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:10:12.033 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:10:12.034 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
May 26 23:10:26.995 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:10:26.995 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 122.698706144s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:35.923159977Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:37.923210778Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:39.923273759Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:41.923303571Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:43.923359563Z","hostname":"oxz_switch","pid":2290}
May 26 23:10:45.450 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 974, tv_nsec: 664471268 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:10:45.451 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:10:45.452 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:45.924360296Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:47.924420648Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:49.924454528Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:51.925503391Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:53.92553603Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:55.925572127Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:57.925664776Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:10:59.925719026Z","hostname":"oxz_switch","pid":2290}
May 26 23:11:00.412 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:11:00.413 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 156.20691387s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:01.925764143Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:03.926789984Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:05.926829677Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:07.926888018Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:09.926936037Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:11.926992287Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:13.927031786Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:29.947306177Z","hostname":"oxz_switch","pid":2290}
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:32.097651205Z","hostname":"oxz_switch","pid":2290}
May 26 23:11:32.769 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1022, tv_nsec: 110489708 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:11:32.771 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:11:32.779 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:11:42.193259212Z","hostname":"oxz_switch","pid":2290}
May 26 23:11:47.750 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 26 23:11:47.751 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 203.665627761s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor buskin@fe80::9025:5bff:fe00:1b53 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-26T23:12:35.350752664Z","hostname":"oxz_switch","pid":2290}
May 26 23:13:11.233 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1120, tv_nsec: 602772102 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:13:11.241 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:13:11.276 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
May 26 23:13:11.291 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task
May 26 23:13:11.292 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): client error (Connect): tcp connect error: Connection refused (os error 146), delay: 287.206093415s, count: 9, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 26 23:14:34.491 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0de351d8-f06d-4422-b635-21cd2ecc6fd2.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("38fcecd4-6efe-4446-8328-b136a9ae9128.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("c235f50a-be6b-4b6e-ab3a-83c43a497bee.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1203, tv_nsec: 865052086 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 26 23:14:34.491 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
May 26 23:14:34.495 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
May 26 23:14:34.552 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "79c96596-2f7d-4c25-b940-8efd0c4033f1", "content-length": "41", "date": "Tue, 26 May 2026 23:14:34 GMT"} }), component: producer-registration-task
May 26 23:14:34.553 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
May 26 23:14:42.107 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:1025
May 26 23:14:42.115 INFO request completed, latency_us: 6648, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 90c4cd41-9e6f-45d2-8ebc-33f84e2392c0, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:43.108 INFO request completed, latency_us: 699, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 754db562-ce25-4fd2-b507-25e939496e33, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:44.108 INFO request completed, latency_us: 587, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: cc484bf3-4637-4a9b-820d-d5ede1030c6d, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:45.107 INFO request completed, latency_us: 587, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 7a2c1c7b-1d99-4c78-9c3c-f82445c1c7ce, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:46.108 INFO request completed, latency_us: 586, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 347a9d77-7a0d-403d-9e2e-77a0f6bc712f, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:47.108 INFO request completed, latency_us: 718, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 5a63b5bf-9dfe-4d3b-a9bb-73e6f10ac8d3, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:48.108 INFO request completed, latency_us: 606, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: fcc7783f-8620-4f4d-8bcb-26148c269073, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:49.107 INFO request completed, latency_us: 598, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: bb3ab4d6-a2b7-4869-a35b-fda827e2992d, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:50.109 INFO request completed, latency_us: 1128, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 8854a71b-80d2-4d93-9c8c-7968b97aca9a, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:51.107 INFO request completed, latency_us: 585, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 825e31fa-85f5-4158-8db0-c378ba96cafe, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:52.107 INFO request completed, latency_us: 593, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 249db841-1b4a-4cac-af33-f28f3c1061c6, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:53.108 INFO request completed, latency_us: 721, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: cca8f35a-59dc-4a96-a15c-df69f31a5f87, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:54.109 INFO request completed, latency_us: 1720, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 7b669aa8-56d9-4770-89c2-ad703a4df428, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:55.107 INFO request completed, latency_us: 587, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: bb55e06f-d4c8-431b-870e-29985fbeedf3, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:56.109 INFO request completed, latency_us: 1187, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: db77700b-8df7-48b4-ae9e-a89d8209cd78, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:57.107 INFO request completed, latency_us: 615, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 2381accf-a68b-479a-9d5d-26626c04eaea, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:58.109 INFO request completed, latency_us: 1057, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: 8dbc0e04-7b52-4674-8dc3-98dba7e8492f, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:14:59.107 INFO request completed, latency_us: 595, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: b35799e1-7817-4cda-84ba-3bcfb265542c, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:15:00.108 INFO request completed, latency_us: 624, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: e761b66f-6960-438c-8e1b-b54bc3ba4fc2, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867
May 26 23:15:01.108 INFO request completed, latency_us: 738, response_code: 200, uri: /5ff07680-a497-4f4f-b278-8c9955051423, method: GET, req_id: b55c817d-76fd-4cf1-a9b0-23aff39050c7, remote_addr: [fd00:1122:3344:101::d]:65143, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:867