[ May 27 20:57:35 Disabled. ]
[ May 27 20:57:35 Rereading configuration. ]
[ May 27 20:57:36 Enabled. ]
[ May 27 20:57:36 Rereading configuration. ]
[ May 27 20:57:36 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ May 27 20:57:36 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:36.743802828Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:36.780959065Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-05-27T20:57:36.781010066Z","hostname":"oxz_switch","pid":2287}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:36.781070619Z","hostname":"oxz_switch","pid":2287}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:36.781907226Z","hostname":"oxz_switch","pid":2287}
{"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:36.782465115Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:37.78104098Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:38.781118434Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:39.821670056Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:40.821767431Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:41.867551377Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:42.867643812Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:43.867719006Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:44.867845403Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:45.867950408Z","hostname":"oxz_switch","pid":2287}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:46.868029863Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::3c4f:8ff:fe11:2346 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:47.868151999Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-05-27T20:57:47.86818826Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] nbr is fe80::10cb:66ff:fe8b:39fc@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:47.86880249Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-05-27T20:57:47.868824011Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::3c4f:8ff:fe11:2346]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:47.868954995Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:47.86907179Z","hostname":"oxz_switch","pid":2287}
[ May 27 20:57:48 Rereading configuration. ]
[ May 27 20:57:48 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=213
+ getopts c: o
+ [[ -n 213 ]]
+ pgrep -c 213
+ PARENT_PID=2286
+ [[ -n 2286 ]]
+ pgrep -P 2286
+ CHILD_PID=2287
+ [[ -n 2287 ]]
+ kill -USR1 2287
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:48.26722373Z","hostname":"oxz_switch","pid":2287}
[ May 27 20:57:48 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:48.26901766Z","hostname":"oxz_switch","pid":2287}
May 27 20:57:48.269 DEBG registered DTrace probes
May 27 20:57:48.270 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
May 27 20:57:48.270 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328
May 27 20:57:48.270 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
May 27 20:57:48.270 DEBG Nexus IP not provided, will use DNS to resolve it
May 27 20:57:48.270 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/internal-dns/resolver/src/resolver.rs:144
May 27 20:57:48.270 DEBG starting producer registration task
May 27 20:57:48.270 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 78e45f3a-93dc-4bce-bdaf-6dab436238aa, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:283
May 27 20:57:48.271 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
May 27 20:57:48.272 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1902, tv_nsec: 225230695 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:57:48.272 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:57:48.274 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Net {\n                addr: 0.0.0.0,\n                width: 0,\n            },\n        ),\n        boundary_addr: fdad:65eb:fe26:a38e::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.482211398Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:57:49.871090821Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.872157116Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-05-27T20:57:49.872212068Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::10cb:66ff:fe8b:39fc on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.872680103Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::10cb:66ff:fe8b:39fc on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.873909804Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::10cb:66ff:fe8b:39fc on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.874270766Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::10cb:66ff:fe8b:39fc on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.874494734Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::10cb:66ff:fe8b:39fc on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.874833645Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-05-27T20:57:49.875121955Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-05-27T20:57:49.875140825Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:03.177 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:58:03.177 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 15.00480473s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 27 20:58:03.348 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1917, tv_nsec: 399508096 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:58:03.349 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:58:03.350 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:06.07568192Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:08.075723686Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:10.075755881Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:12.075807723Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:14.075872048Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:16.075920271Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:18.075930013Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:18.619 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:58:18.619 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 30.178842233s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
May 27 20:58:18.954 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1932, tv_nsec: 737838166 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:58:18.954 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:58:18.955 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:20.075935786Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:22.075996614Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:24.076081382Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:26.076086074Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:28.076128333Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:30.076169242Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:32.07623411Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:33.916 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:58:33.916 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 45.517102911s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:34.076256818Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:35.250 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1949, tv_nsec: 79095955 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:58:35.251 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:58:35.252 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:36.076303036Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:38.076342014Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:40.076367361Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:42.076426568Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:44.076465145Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:46.076482851Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:48.076536058Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:50.076566625Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:50.212 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:58:50.212 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 61.857534923s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:52.076598262Z","hostname":"oxz_switch","pid":2287}
May 27 20:58:53.052 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1966, tv_nsec: 929119170 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:58:53.052 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:58:53.053 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:54.076655519Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:56.076683915Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:58:58.07670331Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:00.076760647Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:02.076796624Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:04.07683437Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:06.076897609Z","hostname":"oxz_switch","pid":2287}
May 27 20:59:08.014 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:59:08.014 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 79.707965221s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:08.076922887Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:10.076978035Z","hostname":"oxz_switch","pid":2287}
May 27 20:59:12.066 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1985, tv_nsec: 995564249 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:59:12.067 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:59:12.068 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:12.077041413Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:14.077077551Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:16.077118348Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:18.077158726Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:20.077198304Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:22.077243312Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:24.077286819Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:26.077336237Z","hostname":"oxz_switch","pid":2287}
May 27 20:59:27.028 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:59:27.028 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 98.774461342s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:28.077400394Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:30.077428681Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:32.077452838Z","hostname":"oxz_switch","pid":2287}
May 27 20:59:33.758 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("44cb9ec0-dc86-4fc8-b845-9f7a3f3236cb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("816dd9ca-b0e8-4c88-8f90-0565f3cfdf4c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9896502c-ee75-44fe-a03d-86a8ef5b60e5.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 2007, tv_nsec: 746535165 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
May 27 20:59:33.759 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
May 27 20:59:33.760 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:34.077492355Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:36.077538797Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:38.077635763Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:40.0776755Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:42.077705187Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:44.077716753Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:46.07776502Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:48.077818718Z","hostname":"oxz_switch","pid":2287}
May 27 20:59:48.721 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
May 27 20:59:48.721 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out, delay: 120.526672639s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/dfcd9ac/oximeter/producer/src/lib.rs:379
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:50.077872535Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:52.077934722Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:54.077955608Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:56.077993546Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T20:59:58.078041202Z","hostname":"oxz_switch","pid":2287}
{"msg":"[tfportrear0_0] neighbor sock@fe80::10cb:66ff:fe8b:39fc missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-05-27T21:00:00.07809864Z","hostname":"oxz_switch","pid":2287}