[ Aug 28 00:51:38 Disabled. ] [ Aug 28 00:51:38 Rereading configuration. ] [ Aug 28 00:51:39 Enabled. ] [ Aug 28 00:51:39 Rereading configuration. ] [ Aug 28 00:51:39 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Aug 28 00:51:39 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:39.205593223Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:39.225130349Z","hostname":"oxz_switch","pid":2244} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2024-08-28T00:51:39.225294494Z","hostname":"oxz_switch","pid":2244} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:39.225311034Z","hostname":"oxz_switch","pid":2244} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:39.226267381Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:40.225224287Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:41.225284334Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:42.22535272Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:43.225421678Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:44.225510755Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:45.225580862Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:46.225625328Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:47.226038085Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:48.22606069Z","hostname":"oxz_switch","pid":2244} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:49.226126668Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] sm initialized with addr fe80::6c89:1aff:fe36:89eb on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:50.226217935Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2024-08-28T00:51:50.226244756Z","hostname":"oxz_switch","pid":2244} [ Aug 28 00:51:50 Rereading configuration. ] [ Aug 28 00:51:50 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 215"). ] + CONTRACT='' + getopts c: o + CONTRACT=215 + getopts c: o + [[ -n 215 ]] + pgrep -c 215 + PARENT_PID=2242 + [[ -n 2242 ]] + pgrep -P 2242 + CHILD_PID=2244 + [[ -n 2244 ]] + kill -USR1 2244 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:50.508414202Z","hostname":"oxz_switch","pid":2244} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:50.509978537Z","hostname":"oxz_switch","pid":2244} [ Aug 28 00:51:50 Method "refresh" exited with status 0. ] Aug 28 00:51:50.510 DEBG registered DTrace probes Aug 28 00:51:50.511 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 28 00:51:50.511 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205 Aug 28 00:51:50.511 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 28 00:51:50.511 DEBG Nexus IP not provided, will use DNS to resolve it Aug 28 00:51:50.511 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/internal-dns/src/resolver.rs:73 Aug 28 00:51:50.511 DEBG starting producer registration task Aug 28 00:51:50.511 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 16dbca8f-0464-4c22-aca8-52b882ce8c8c, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:283 Aug 28 00:51:50.511 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fde4:9bf9:2992:9055::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:51.772341615Z","hostname":"oxz_switch","pid":2244} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fde4:9bf9:2992:9055::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:51.77250624Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fde4:9bf9:2992:9055::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:51.772555071Z","hostname":"oxz_switch","pid":2244} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fde4:9bf9:2992:9055::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:51.772923892Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fde4:9bf9:2992:9055::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:51:51.772934702Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] nbr is fe80::f006:acff:feb1:b1ca@buskin server","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.226671678Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2024-08-28T00:51:52.226707429Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] exchange: listening on [fe80::6c89:1aff:fe36:89eb]:56797","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.226834132Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.226965616Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.228669615Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2024-08-28T00:51:52.228713656Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::f006:acff:feb1:b1ca on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.270393531Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::f006:acff:feb1:b1ca on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.271701319Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::f006:acff:feb1:b1ca on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.271878434Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:9fb2:: -> fe80::f006:acff:feb1:b1ca on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.272864902Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::f006:acff:feb1:b1ca on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.272999256Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-28T00:51:52.273597883Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2024-08-28T00:51:52.30337452Z","hostname":"oxz_switch","pid":2244} Aug 28 00:51:55.515 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ee19452-709a-4abd-a238-7af80f61d166.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("a5227819-155d-4a08-8e71-bd8a327ad739.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("f840e071-026c-488d-b1c9-2c4d1ce3e74d.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("f840e071-026c-488d-b1c9-2c4d1ce3e74d.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 779, tv_nsec: 134076531 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Aug 28 00:51:55.516 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::c]:12221, component: producer-registration-task Aug 28 00:51:55.547 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:10.710660138Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:11.025 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:52:11.026 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 141.056877ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 Aug 28 00:52:11.197 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:12.711676628Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:14.712690067Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:16.713701081Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:18.714712115Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:20.715720708Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:22.716750711Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:24.717768315Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:26.156 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:52:26.157 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 646.668548ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:26.718775908Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:26.833 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:28.719786561Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:30.720801184Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:32.721821197Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:34.72282919Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:36.723841163Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:38.724859237Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:40.72586257Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:41.793 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:52:41.794 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 1.441316662s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:42.726874103Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:43.262 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:44.727903037Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:46.72890697Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:48.729925203Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:50.730954537Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:52.73196083Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:54.732961423Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:56.733965936Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:58.223 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:52:58.224 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 1.196867539s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:52:58.73498287Z","hostname":"oxz_switch","pid":2244} Aug 28 00:52:59.448 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:00.735989083Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:02.737007186Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:04.73801789Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:06.739043783Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:08.740053827Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:10.74107147Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:12.742076603Z","hostname":"oxz_switch","pid":2244} Aug 28 00:53:14.409 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:53:14.410 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 5.151771528s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:14.743089627Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:16.74410831Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:18.745125714Z","hostname":"oxz_switch","pid":2244} Aug 28 00:53:19.579 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:20.746156908Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:22.747163741Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:24.748177544Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:26.749195728Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:28.750205791Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:30.751222845Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:32.752220848Z","hostname":"oxz_switch","pid":2244} Aug 28 00:53:34.539 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:53:34.540 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 6.369444799s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:34.753243251Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:36.754259515Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:38.755286769Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:40.756295052Z","hostname":"oxz_switch","pid":2244} Aug 28 00:53:40.923 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:42.757314916Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:44.758323789Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:46.759340442Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:48.760354506Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:50.761364959Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:52.762379483Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:54.763409357Z","hostname":"oxz_switch","pid":2244} Aug 28 00:53:55.883 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:53:55.884 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 21.933257979s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:56.76441831Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:53:58.765432873Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:00.766456607Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:02.76746484Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:04.768479274Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:06.769495077Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:08.770507891Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:10.771533894Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:12.772555248Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:14.773570441Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:16.774594245Z","hostname":"oxz_switch","pid":2244} Aug 28 00:54:17.790 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:18.775598728Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:20.776615332Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:22.777624385Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:24.778635328Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:26.807793845Z","hostname":"oxz_switch","pid":2244} {"msg":"[tfportrear0_0] neighbor buskin@fe80::f006:acff:feb1:b1ca missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-28T00:54:28.807158206Z","hostname":"oxz_switch","pid":2244} Aug 28 00:54:32.763 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:54:32.764 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 26.475227525s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 Aug 28 00:54:59.293 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 28 00:55:14.306 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 28 00:55:14.309 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 32.392341383s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 Aug 28 00:55:46.739 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 28 00:55:46.745 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: hyper::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task Aug 28 00:55:46.746 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)", delay: 118.281704416s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424 Aug 28 00:57:45.058 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 28 00:57:45.064 DEBG client response, result: Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, status: 503, headers: {"content-type": "application/json", "x-request-id": "f8d1cf27-774f-4841-b5ad-338ec162530f", "content-length": "133", "date": "Wed, 28 Aug 2024 00:57:45 GMT"} }), component: producer-registration-task Aug 28 00:57:45.067 WARN failed to register as a producer with Nexus, will retry, error: "Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8d1cf27-774f-4841-b5ad-338ec162530f\", \"content-length\": \"133\", \"date\": \"Wed, 28 Aug 2024 00:57:45 GMT\"}; value: Error { error_code: Some(\"ServiceNotAvailable\"), message: \"Service Unavailable\", request_id: \"f8d1cf27-774f-4841-b5ad-338ec162530f\" }", delay: 221.850373176s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/66ac7b3/oximeter/producer/src/lib.rs:424