[ Jul 17 01:53:28 Disabled. ]
[ Jul 17 01:53:28 Rereading configuration. ]
[ Jul 17 01:53:28 Enabled. ]
[ Jul 17 01:53:28 Rereading configuration. ]
[ Jul 17 01:53:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ Jul 17 01:53:28 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:28.921882177Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:28.944263941Z","hostname":"oxz_switch","pid":2295}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-07-17T01:53:28.944303231Z","hostname":"oxz_switch","pid":2295}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:28.944313611Z","hostname":"oxz_switch","pid":2295}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:28.945594429Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:29.944368635Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:30.944439969Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:31.944534752Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:32.944685596Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:33.94484543Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:34.944914513Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:35.944994836Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:36.94509171Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:37.945174283Z","hostname":"oxz_switch","pid":2295}
{"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:38.945255927Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::c42:b5ff:fe50:91b3 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:39.945417991Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2026-07-17T01:53:39.945445801Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] nbr is fe80::c1e:8dff:fef0:3f72@sock server","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:39.946114105Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2026-07-17T01:53:39.946134695Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::c42:b5ff:fe50:91b3]:56797","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:39.946310996Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:39.946411646Z","hostname":"oxz_switch","pid":2295}
[ Jul 17 01:53:40 Rereading configuration. ]
[ Jul 17 01:53:40 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 213"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=213
+ getopts c: o
+ [[ -n 213 ]]
+ pgrep -c 213
+ PARENT_PID=2292
+ [[ -n 2292 ]]
+ pgrep -P 2292
+ CHILD_PID=2295
+ [[ -n 2295 ]]
+ kill -USR1 2295
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:40.369828733Z","hostname":"oxz_switch","pid":2295}
[ Jul 17 01:53:40 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:40.372580669Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:53:40.373 DEBG registered DTrace probes
Jul 17 01:53:40.373 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jul 17 01:53:40.373 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:335
Jul 17 01:53:40.374 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jul 17 01:53:40.374 DEBG Nexus IP not provided, will use DNS to resolve it
Jul 17 01:53:40.374 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/internal-dns/resolver/src/resolver.rs:144
Jul 17 01:53:40.374 DEBG starting producer registration task
Jul 17 01:53:40.374 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 4cb9bc7c-b0b7-4081-9485-9ba54150692e, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:280
Jul 17 01:53:40.374 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
Jul 17 01:53:40.375 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 1.077026ms, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
Jul 17 01:53:40.570 WARN failed to register with Nexus, will retry, error: proto error: no records found for Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, delay: 196.187457ms, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
Jul 17 01:53:41.180 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 676, tv_nsec: 260536216 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:53:41.181 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:53:41.183 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Net {\n                addr: 0.0.0.0,\n                width: 0,\n            },\n        ),\n        boundary_addr: fd05:748b:5597:8528::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.554969403Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] exchange pull: hyper client error: client error (Connect)","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:41.948212602Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.949267478Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2026-07-17T01:53:41.949291128Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::c1e:8dff:fef0:3f72 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.950638746Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::c1e:8dff:fef0:3f72 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.951816292Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::c1e:8dff:fef0:3f72 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.952119914Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::c1e:8dff:fef0:3f72 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.952325985Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::c1e:8dff:fef0:3f72 on port Rear(Rear(\"rear0\"))/0","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.952618717Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2026-07-17T01:53:41.952839998Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2026-07-17T01:53:41.953599142Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:53:56.485 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:53:56.485 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 15.810300466s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
Jul 17 01:53:57.314 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 692, tv_nsec: 96230613 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:53:57.315 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:53:57.316 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:53:58.247317894Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:00.2473553Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:02.247414357Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:04.247468264Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:06.24749343Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:08.247521586Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:10.247568222Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:12.247592208Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:12.276 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:54:12.277 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 31.644599876s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:14.2486152Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:15.259 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 710, tv_nsec: 89612830 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:54:15.260 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:54:15.261 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:16.248682746Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:18.248733742Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:20.248782889Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:22.248822735Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:24.248896931Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:26.248943737Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:28.249011913Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:30.222 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:54:30.222 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 49.638522296s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:30.24905551Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:32.249113956Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:34.249146422Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:35.267 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 730, tv_nsec: 152077137 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:54:35.268 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:54:35.269 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:36.249202678Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:38.249262485Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:40.249307871Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:42.249361587Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:44.249411833Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:46.249471889Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:48.249509516Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:50.230 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:54:50.230 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 69.70045985s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:50.249551052Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:52.249613188Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:54.249620894Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:54:54.949 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 749, tv_nsec: 887108648 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:54:54.950 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:54:54.951 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:56.249670341Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:54:58.249694637Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:00.249716213Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:02.249779159Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:04.249842116Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:06.249885372Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:08.249944268Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:55:09.912 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:55:09.912 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 89.435461611s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:10.250024104Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:12.25003677Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:14.250097847Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:16.250135213Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:18.250196619Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:20.250207879Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:55:20.414 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 775, tv_nsec: 420284218 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:55:20.414 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:55:20.415 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:22.250294881Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:24.250328377Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:26.250361564Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:28.25043369Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:30.250491466Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:32.250549853Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:34.250597259Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:55:35.376 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:55:35.376 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 114.968150538s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:36.250638455Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:38.250714561Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:40.250781778Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:42.250847744Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:44.250889839Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:46.250940475Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:48.250963662Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:55:50.251014108Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:56:01.215 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 816, tv_nsec: 332356323 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:56:01.216 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:56:01.217 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
Jul 17 01:56:16.177 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:56:16.177 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 155.880083892s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:24.252370296Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:26.252391102Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:28.252434418Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:30.252491654Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:32.252529081Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:34.252582877Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:36.252591993Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:38.252635679Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:40.252708895Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:42.252750722Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:44.252774478Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:46.252830174Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:48.252869688Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:56:56.257930603Z","hostname":"oxz_switch","pid":2295}
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:57:00.27813431Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:57:05.790 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 881, tv_nsec: 28552994 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:57:05.790 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:57:05.791 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::c1e:8dff:fef0:3f72 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2026-07-17T01:57:12.530417964Z","hostname":"oxz_switch","pid":2295}
Jul 17 01:57:20.792 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jul 17 01:57:20.792 WARN failed to register with Nexus, will retry, error: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out, delay: 220.576618915s, count: 9, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/d6fba55/oximeter/producer/src/lib.rs:376
Jul 17 01:59:38.013 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("46ba6f5a-71a4-404a-b4c5-53ff6dff8565.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("94b48d02-0980-4e18-94c9-12f8193ce7a1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("9a95a123-1e6d-4334-b988-7e526a502d1b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 1033, tv_nsec: 245422722 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jul 17 01:59:38.016 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
Jul 17 01:59:38.022 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
Jul 17 01:59:38.090 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "ac2fc70b-d31e-4438-9558-b1b37f6cd885", "content-length": "41", "date": "Fri, 17 Jul 2026 01:59:38 GMT"} }), component: producer-registration-task
Jul 17 01:59:38.090 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
Jul 17 01:59:42.523 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:1057
Jul 17 01:59:42.531 INFO request completed, latency_us: 4655, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 700baf32-3ea1-4c15-a6ad-f2d3436ea51a, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:43.524 INFO request completed, latency_us: 681, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: d47c4458-be6d-458f-a60d-c49d42acd858, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:44.524 INFO request completed, latency_us: 1066, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 429e93bf-ef43-4fd6-8102-69d73fcfbbe8, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:45.524 INFO request completed, latency_us: 804, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: a1eef2c8-56ae-406e-99b6-29b910249f65, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:46.524 INFO request completed, latency_us: 1146, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 333695ce-f8b1-455c-959e-3d60967b6be6, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:47.524 INFO request completed, latency_us: 1112, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: ae977dca-f330-4453-949c-e150b6442fa2, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:48.524 INFO request completed, latency_us: 1092, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: b47afbf8-c095-4a19-bba8-e7ab0c06e284, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:49.523 INFO request completed, latency_us: 866, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: f3cbe459-bbd5-49ba-99ea-5f7e2da7bace, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:50.524 INFO request completed, latency_us: 923, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: b5a7cbc6-e928-4da4-9c18-a93830434fce, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:51.524 INFO request completed, latency_us: 719, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 07b26d79-c12a-4ad5-b8a0-ae6f85ad0b99, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:52.524 INFO request completed, latency_us: 1029, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 8d7cbcbb-5b17-42ca-94f5-2adc8038f758, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:53.524 INFO request completed, latency_us: 868, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: ecd1ed79-35a8-4322-b48b-d21414eeddc2, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:54.524 INFO request completed, latency_us: 697, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 6bfb2554-1bf2-4c1b-b98b-d180d58fb5db, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:55.524 INFO request completed, latency_us: 1117, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 985f10d6-a916-41c2-b6a4-efc5b02e16a9, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:56.524 INFO request completed, latency_us: 638, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 4f205348-7e56-4c4c-9101-083192585d5a, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:57.524 INFO request completed, latency_us: 679, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 9d2d52a7-94af-42d4-b645-910d0ec39123, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:58.524 INFO request completed, latency_us: 721, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 851021f7-db47-4401-8d2e-4f8ef973ee70, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874
Jul 17 01:59:59.524 INFO request completed, latency_us: 571, response_code: 200, uri: /4cb9bc7c-b0b7-4081-9485-9ba54150692e, method: GET, req_id: 86847ab7-ed5a-4692-8568-fb76479bb16c, remote_addr: [fd00:1122:3344:101::d]:62834, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.17.1/src/server.rs:874