[ Jun 20 21:23:33 Disabled. ]
[ Jun 20 21:23:33 Rereading configuration. ]
[ Jun 20 21:23:46 Rereading configuration. ]
[ Jun 20 21:23:46 Rereading configuration. ]
[ Jun 20 21:23:46 Enabled. ]
[ Jun 20 21:23:46 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mg-ddm/pkg/ddm_method_script.sh &"). ]
[ Jun 20 21:23:46 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.822201232Z","hostname":"oxz_switch","pid":4236}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.840816885Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] sm initialized with addr fe80::50b4:daff:fef2:43f4 on if tfportrear0_0 index 3","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.841550833Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] starting discovery handler","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.841623505Z","hostname":"oxz_switch","pid":4236}
{"msg":"admin: listening on [::1]:8000","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.842236481Z","hostname":"oxz_switch","pid":4236}
[ Jun 20 21:23:48 Rereading configuration. ]
[ Jun 20 21:23:48 Executing refresh method ("/opt/oxide/mg-ddm/pkg/refresh_ctrun_child.sh -c 445"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=445
+ getopts c: o
+ [[ -n 445 ]]
+ pgrep -c 445
+ PARENT_PID=4235
+ [[ -n 4235 ]]
+ pgrep -P 4235
+ CHILD_PID=4236
+ [[ -n 4236 ]]
+ kill -USR1 4236
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.153308628Z","hostname":"oxz_switch","pid":4236}
[ Jun 20 21:23:48 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.154908809Z","hostname":"oxz_switch","pid":4236}
{"msg":"resolving nexus","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.155312869Z","hostname":"oxz_switch","pid":4236}
{"msg":"new DNS resolver","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.155410022Z","hostname":"oxz_switch","pid":4236,"addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"[tfportrear0_0] nbr is fe80::5860:2fff:fece:cdc0@sock server","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.842128668Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] transition solicit -> exchange","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:48.84219931Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] exchange: listening on [fe80::50b4:daff:fef2:43f4]:56797","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.842483737Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] waiting for exchange server to start","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.842688922Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] sending 5 routes to dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.844667892Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] sending to dpd host=localhost port=12224","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:48.844680702Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:2:: -> fe80::5860:2fff:fece:cdc0 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.883367625Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:101:: -> fe80::5860:2fff:fece:cdc0 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.884569986Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] adding route fdb0:18c0:4d0d:a02a:: -> fe80::5860:2fff:fece:cdc0 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.8847328Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:3:: -> fe80::5860:2fff:fece:cdc0 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.884877604Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] adding route fd00:1122:3344:1:: -> fe80::5860:2fff:fece:cdc0 on port Rear(RearPort(0))/0","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.885003117Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] removing routes 0 from dendrite","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.886147096Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] redistributing update to 0 peers","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:48.915377969Z","hostname":"oxz_switch","pid":4236}
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Prefix {\n                addr: 0.0.0.0,\n                len: 0,\n            },\n        ),\n        boundary_addr: fd36:3b21:3616:2cac::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:49.229183061Z","hostname":"oxz_switch","pid":4236}
{"msg":"advertise tunnel: {\n    TunnelOrigin {\n        overlay_prefix: V4(\n            Ipv4Prefix {\n                addr: 0.0.0.0,\n                len: 0,\n            },\n        ),\n        boundary_addr: fd36:3b21:3616:2cac::1,\n        vni: 99,\n        metric: 18446744073709551615,\n    },\n}","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:49.229625402Z","hostname":"oxz_switch","pid":4236}
{"msg":"resolving nexus failed, no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN } waiting 85.876128ms","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:23:53.157666973Z","hostname":"oxz_switch","pid":4236}
{"msg":"lookup_socket_v6 srv","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:53.245289419Z","hostname":"oxz_switch","pid":4236,"response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"65b469d2-6eb7-4e46-8276-ad6d27b67bf9.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"8f64d0bd-6f7c-418c-a532-2ff9143ec947.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"8f64d0bd-6f7c-418c-a532-2ff9143ec947.host.control-plane.oxide.internal.\"), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 639, tv_nsec: 808200905 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
Jun 20 21:23:53.247 DEBG registered DTrace probes
Jun 20 21:23:53.247 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jun 20 21:23:53.247 INFO listening, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:196
Jun 20 21:23:53.247 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:8001, component: dropshot
Jun 20 21:23:53.247 DEBG Nexus IP provided explicitly, will use it, addr: [fd00:1122:3344:101::a]:12221
Jun 20 21:23:53.247 DEBG starting producer registration task
Jun 20 21:23:53.247 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:8001, producer_id: 6bdcfeaf-3a27-40c7-86ba-9159225ea084, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:276
Jun 20 21:23:53.247 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task
Jun 20 21:23:53.247 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jun 20 21:23:53.277 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:07.882 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:07.883 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 218.769639ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:08.132 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:23.133 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:23.134 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 605.456411ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:23.770 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:37.172510904Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:39.173531477Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:24:39.487 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:39.488 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 968.513634ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:40.483 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:41.174561381Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:43.175564153Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:45.176598578Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:47.177630351Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:49.178642194Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:51.179664638Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:53.180681091Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:55.181701845Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:24:55.444 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:55.445 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 1.577978383s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:57.049 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:57.182729028Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:24:59.183746792Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:01.184769656Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:03.185806839Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:05.186819442Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:07.187852705Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:09.188867039Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:11.189894812Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:25:12.009 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:12.010 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 2.558206748s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:13.190928547Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:25:14.591 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:15.191945589Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:17.192975013Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:19.193981655Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:21.195004608Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:23.196011192Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:25.197024486Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:27.19805177Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:29.199082024Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:25:29.551 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:29.552 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 11.76066324s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:31.200100146Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:33.201130481Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:35.202137304Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:37.203146159Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:39.204181184Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:41.205187678Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:25:41.311 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:43.206219443Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:45.207215908Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:47.208243882Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:49.209285317Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:51.21029854Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:53.211330695Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:55.21234425Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:25:56.272 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:56.273 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 17.268059697s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:57.213367385Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:25:59.214379418Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:01.215396433Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:03.216433708Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:05.217437312Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:07.218458195Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:09.21946163Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:11.220492754Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:13.221489649Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:26:13.525 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:15.222514752Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:17.223524547Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:19.22453348Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:21.225565966Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:23.226571309Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:25.227608966Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:27.228626838Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:26:28.485 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:26:28.486 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 35.557089896s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:29.229630001Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:31.230659895Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:33.23167431Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:35.232692903Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:37.233721168Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:39.234718442Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:41.235755486Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:43.23677258Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:45.237802094Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:47.238832828Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:49.239844962Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:51.240887197Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:53.24189175Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:55.242918593Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:57.243954588Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:26:59.244961782Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:01.246000956Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:03.247004579Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:27:03.977 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:05.248022043Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:07.249064714Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:09.25006951Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:11.251101724Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:13.252104277Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:15.253125672Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:17.254156565Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:27:18.937 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:27:18.939 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 68.068370897s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:19.25516643Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:21.256200184Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:23.257219178Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:25.258239042Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:27.259256365Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:27:29.272408208Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:28:12.201737921Z","hostname":"oxz_switch","pid":4236}
{"msg":"[tfportrear0_0] neighbor sock@fe80::5860:2fff:fece:cdc0 missed solicit interval","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:28:16.232986814Z","hostname":"oxz_switch","pid":4236}
Jun 20 21:28:27.049 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:28:42.050 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:28:42.054 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 178.091766715s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417