[ Aug 1 05:10:14 Disabled. ] [ Aug 1 05:10:14 Rereading configuration. ] [ Aug 1 05:10:15 Enabled. ] [ Aug 1 05:10:15 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Aug 1 05:10:15 Method "start" exited with status 0. ] [ Aug 1 05:10:15 Rereading configuration. ] [ Aug 1 05:10:15 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 215"). ] + CONTRACT='' + getopts c: o + CONTRACT=215 + getopts c: o + [[ -n 215 ]] + pgrep -c 215 + PARENT_PID=2270 + [[ -n 2270 ]] + pgrep -P 2270 + CHILD_PID=2272 + [[ -n 2272 ]] + kill -USR1 2272 [ Aug 1 05:10:15 Method "refresh" exited with status 0. ] [ Aug 1 05:10:15 Stopping because all processes in service exited. ] [ Aug 1 05:10:15 Executing stop method (:kill). ] [ Aug 1 05:10:15 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ] [ Aug 1 05:10:15 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:15.534398443Z","hostname":"oxz_switch","pid":2290} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:15.554965394Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:15.555410205Z","hostname":"oxz_switch","pid":2290} {"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:15.556112554Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:16.555554608Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:17.555631519Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:18.55571065Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:19.555792231Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:20.555897653Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:21.556034395Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:22.55626514Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:23.556209408Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:24.556287599Z","hostname":"oxz_switch","pid":2290} {"msg":"[] failed to get IPv6 address for interface tfportrear0_0/ll: not found: tfportrear0_0/ll","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:25.55636373Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sm initialized with addr fe80::a87d:ddff:fe2a:1b5f on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:26.556552944Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:10:26.556594275Z","hostname":"oxz_switch","pid":2290} [ Aug 1 05:10:26 Rereading configuration. ] [ Aug 1 05:10:26 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 218"). ] + CONTRACT='' + getopts c: o + CONTRACT=218 + getopts c: o + [[ -n 218 ]] + pgrep -c 218 + PARENT_PID=2289 + [[ -n 2289 ]] + pgrep -P 2289 + CHILD_PID=2290 + [[ -n 2290 ]] + kill -USR1 2290 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:26.884538232Z","hostname":"oxz_switch","pid":2290} [ Aug 1 05:10:26 Method "refresh" exited with status 0. ] {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:26.886695458Z","hostname":"oxz_switch","pid":2290} {"msg":"resolving nexus","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:26.88714863Z","hostname":"oxz_switch","pid":2290} {"msg":"new DNS resolver","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:26.887248472Z","hostname":"oxz_switch","pid":2290,"addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Underlay({Ipv6Net { addr: fd41:d724:2c93:8ec3::1, width: 64 }}))","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:27.963553271Z","hostname":"oxz_switch","pid":2290} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd41:d724:2c93:8ec3::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:27.963717635Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] admin event in solicit state: Announce(Tunnel({TunnelOrigin { overlay_prefix: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), boundary_addr: fd41:d724:2c93:8ec3::1, vni: 99, metric: 18446744073709551615 }}))","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:27.963777137Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] nbr is fe80::28cd:8ff:fef6:9905@sock server","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.557141667Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:10:28.55724189Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] exchange: listening on [fe80::a87d:ddff:fe2a:1b5f]:56797","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.557400824Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.557536477Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.559421416Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:10:28.559432706Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.598415923Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.599719126Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.59988216Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.600035695Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.600166798Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:28.601212305Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:10:28.630633345Z","hostname":"oxz_switch","pid":2290} {"msg":"advertise tunnel: {\n TunnelOrigin {\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n boundary_addr: fd41:d724:2c93:8ec3::1,\n vni: 99,\n metric: 18446744073709551615,\n },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:10:29.965060198Z","hostname":"oxz_switch","pid":2290} {"msg":"lookup_socket_v6 srv","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:10:31.890492621Z","hostname":"oxz_switch","pid":2290,"response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"1d5954f0-b510-4a4f-be37-9b6e9a43e8d1.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"2f888246-b273-44e6-ad3e-5bcd759c4926.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"a18d61d4-7a53-4fa9-aa91-9b6d1e0cca06.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"a18d61d4-7a53-4fa9-aa91-9b6d1e0cca06.host.control-plane.oxide.internal.\"), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 2650, tv_nsec: 11399530 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} Aug 01 05:10:31.891 DEBG registered DTrace probes Aug 01 05:10:31.891 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 01 05:10:31.892 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:196 Aug 01 05:10:31.892 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot Aug 01 05:10:31.892 DEBG Nexus IP provided explicitly, will use it, addr: [fd00:1122:3344:101::b]:12221 Aug 01 05:10:31.892 DEBG starting producer registration task Aug 01 05:10:31.892 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: e84fd94e-b907-46f3-9691-af174f5180ba, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:276 Aug 01 05:10:31.892 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task Aug 01 05:10:31.892 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::b]:12221, component: producer-registration-task Aug 01 05:10:31.922 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:46.376952026Z","hostname":"oxz_switch","pid":2290} Aug 01 05:10:46.736 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:10:46.737 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 289.934331ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 Aug 01 05:10:47.058 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:48.377987434Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:50.379002949Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:52.380023285Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:54.381025779Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:56.382026884Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:10:58.383037318Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:00.384044023Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:02.017 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:11:02.019 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 552.117601ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:02.385054078Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:02.600 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:04.386062877Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:06.387071537Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:08.388110103Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:10.389119317Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:12.390141833Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:14.391139697Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:16.392183383Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:17.560 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:11:17.562 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 517.842101ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 Aug 01 05:11:18.108 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:18.393186287Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:20.394209562Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:22.395229187Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:24.396230312Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:26.397254267Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:28.398276152Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:30.399273166Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:32.400300841Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:33.068 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:11:33.070 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 1.680840295s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:34.401309635Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:34.777 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:36.402334351Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:38.403364096Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:40.40436359Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:42.405371815Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:11:44.406380019Z","hostname":"oxz_switch","pid":2290} Aug 01 05:11:49.737 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:11:49.738 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 4.434548538s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 Aug 01 05:11:54.192 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 01 05:12:09.152 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:12:09.153 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 9.524899581s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 Aug 01 05:12:18.683 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:12:22.430097116Z","hostname":"oxz_switch","pid":2290} Aug 01 05:12:33.679 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:12:33.680 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 22.007586107s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:12:34.471675657Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:12:47.050095811Z","hostname":"oxz_switch","pid":2290} Aug 01 05:12:55.721 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 01 05:13:10.728 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task Aug 01 05:13:10.733 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out", delay: 34.462300783s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417 {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:13:11.041950376Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] neighbor sock@fe80::28cd:8ff:fef6:9905 expire","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:13:17.110730924Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] expiring peer fe80::28cd:8ff:fef6:9905 due to discovery event","v":0,"name":"slog-rs","level":40,"time":"2024-08-01T05:13:17.2023477Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 5 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:17.202380731Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] nbr is fe80::28cd:8ff:fef6:9905@sock server","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:17.277576512Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] failed to remove tunnel routes: {} OPTE driver is not attached","v":0,"name":"slog-rs","level":50,"time":"2024-08-01T05:13:18.430804267Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing expire to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:13:18.43089258Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:13:18.433746854Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] exchange: listening on [fe80::a87d:ddff:fe2a:1b5f]:56797","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.434490413Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.435990921Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.447370455Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:13:18.447393886Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.52499835Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.528063628Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.528266063Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.528445298Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::28cd:8ff:fef6:9905 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.529041343Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-08-01T05:13:18.530245374Z","hostname":"oxz_switch","pid":2290} {"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2024-08-01T05:13:18.622518977Z","hostname":"oxz_switch","pid":2290} Aug 01 05:13:45.241 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Aug 01 05:13:45.244 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::b)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: hyper::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task Aug 01 05:13:45.245 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)", delay: 95.828700418s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417