[ Dec 19 20:26:49 Disabled. ] [ Dec 19 20:26:49 Rereading configuration. ] [ Dec 19 20:26:51 Enabled. ] [ Dec 19 20:26:51 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Dec 19 20:26:51 Method "start" exited with status 0. ] [ Dec 19 20:26:51 Rereading configuration. ] [ Dec 19 20:26:51 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ] + CONTRACT='' + getopts c: o + CONTRACT=214 + getopts c: o + [[ -n 214 ]] + pgrep -c 214 + PARENT_PID=2292 + [[ -n 2292 ]] + pgrep -P 2292 + CHILD_PID=2293 + [[ -n 2293 ]] + kill -USR1 2293 [ Dec 19 20:26:51 Method "refresh" exited with status 0. ] [ Dec 19 20:26:51 Stopping because all processes in service exited. ] [ Dec 19 20:26:51 Executing stop method (:kill). ] [ Dec 19 20:26:51 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Dec 19 20:26:51 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.192626322Z","hostname":"oxz_switch","pid":2320} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.213087558Z","hostname":"oxz_switch","pid":2320} {"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.213291022Z","hostname":"oxz_switch","pid":2320} {"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.213350004Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2024-12-19T20:26:51.213870756Z","hostname":"oxz_switch","pid":2320} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217181933Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217222394Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217235094Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers/{addr}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217244904Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/clear/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217262004Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217271635Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217282675Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217291485Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217300245Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217311336Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217321626Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217334706Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217344357Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217356507Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217366887Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217378557Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217387887Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217399098Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217410618Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217421208Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217433728Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217443229Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/routers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217464769Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217476169Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.21749132Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.21750093Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.21750979Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/message-history","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217527341Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/omicron/apply","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217537201Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/exported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217551611Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/imported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217561002Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217572502Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/selected","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217583382Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217594632Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.217604683Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"PUT"} {"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.217628083Z","hostname":"oxz_switch","pid":2320} {"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:26:51.217643093Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2024-12-19T20:26:51.218892573Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:51.287742003Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:52.289440675Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:53.290045512Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:54.290601688Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:55.291142353Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:56.291740859Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:57.292435519Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:58.293066095Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:26:59.293605441Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:27:00.294285849Z","hostname":"oxz_switch","pid":2320} {"msg":"failed to ensure TEP address fd5c:3d:9074:1d2c::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-19T20:27:01.294845365Z","hostname":"oxz_switch","pid":2320} [ Dec 19 20:27:02 Rereading configuration. ] [ Dec 19 20:27:02 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 219"). ] + CONTRACT='' + getopts c: o + CONTRACT=219 + getopts c: o + [[ -n 219 ]] + pgrep -c 219 + PARENT_PID=2318 + [[ -n 2318 ]] + pgrep -P 2318 + CHILD_PID=2320 + [[ -n 2320 ]] + kill -USR1 2320 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:27:02.418725337Z","hostname":"oxz_switch","pid":2320} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:27:02.420021157Z","hostname":"oxz_switch","pid":2320} [ Dec 19 20:27:02 Method "refresh" exited with status 0. ] Dec 19 20:27:02.420 DEBG registered DTrace probes Dec 19 20:27:02.421 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Dec 19 20:27:02.421 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197 Dec 19 20:27:02.421 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Dec 19 20:27:02.421 DEBG Nexus IP not provided, will use DNS to resolve it Dec 19 20:27:02.421 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/internal-dns/src/resolver.rs:72 Dec 19 20:27:02.421 DEBG starting producer registration task Dec 19 20:27:02.421 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: a49cc98a-21bf-4a6e-b607-d8c90c5416e4, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:283 Dec 19 20:27:02.422 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:27:02.615489433Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:38354"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:27:02.616165918Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"PUT","req_id":"14212a65-9cdd-4154-9b1e-4994591c0bf3","remote_addr":"[fd00:1122:3344:101::1]:38354","local_addr":"[::]:4676","unit":"api-server","latency_us":425,"response_code":"204"} {"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2024-12-19T20:27:02.616616929Z","hostname":"oxz_switch","pid":2320} {"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:27:02.616631949Z","hostname":"oxz_switch","pid":2320} Dec 19 20:27:17.426 WARN failed to lookup Nexus IP, will retry, error: "request timed out", delay: 247.602705ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:391 Dec 19 20:27:17.675 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("1e313b97-01af-42b4-82f0-7eba0dd97e95.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("266cc060-1a63-449d-bd97-a94590d3a870.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("cc262997-f6b5-431b-9033-f5d126feef2f.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("cc262997-f6b5-431b-9033-f5d126feef2f.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::c))) }], valid_until: Instant { tv_sec: 988, tv_nsec: 232447907 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Dec 19 20:27:17.676 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::b]:12221, component: producer-registration-task Dec 19 20:27:17.708 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:27:32.288 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:27:32.289 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 125.849901ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:27:32.445 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:27:47.404 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:27:47.405 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 706.94038ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:27:48.143 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:28:03.103 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:28:03.104 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 869.757015ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:28:04.003 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:28:21.020 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:28:21.347 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 1.892224177s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:28:23.313 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:28:41.062 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:28:41.834 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 2.179145669s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:28:45.048 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:29:00.050 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 19 20:29:00.056 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 6.814525381s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 19 20:29:06.914 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:29:15.493 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task Dec 19 20:29:15.494 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 22.046268174s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:22.666801075Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:64248"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:22.671788361Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"46812b6a-ba93-4893-8f4d-72fff0c88845","remote_addr":"[fd00:1122:3344:101::b]:64248","local_addr":"[::]:4676","unit":"api-server","latency_us":2990,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:23.965826211Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:46018"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:23.965925923Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"ee8815d2-2dca-4c73-97f6-48ba3cddde66","remote_addr":"[fd00:1122:3344:101::c]:46018","local_addr":"[::]:4676","unit":"api-server","latency_us":24,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:24.289869376Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:52102"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:24.289921117Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"35f542e0-307a-4132-b25c-2b63dab5e2f2","remote_addr":"[fd00:1122:3344:101::a]:52102","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"} Dec 19 20:29:37.572 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 19 20:29:37.586 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "28c32e6b-1a45-400d-ae92-3bb260af46cc", "content-length": "41", "date": "Thu, 19 Dec 2024 20:29:37 GMT"} }), component: producer-registration-task Dec 19 20:29:37.588 DEBG registered with nexus successfully, component: producer-registration-task Dec 19 20:29:37.588 DEBG pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task Dec 19 20:29:51.629 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105 Dec 19 20:29:51.638 INFO request completed, latency_us: 7805, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 0c4d161c-ad86-4c94-a30d-036f48ca2faf, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:52.633 INFO request completed, latency_us: 329, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: f551fae2-93eb-4759-892b-9ffc522e434c, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.667208018Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:55405"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.66730229Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"f6d49b4a-7b6f-4c58-97c0-01655d42bfb6","remote_addr":"[fd00:1122:3344:101::b]:55405","local_addr":"[::]:4676","unit":"api-server","latency_us":25,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.855096907Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:38882"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.855802544Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"GET","req_id":"fb717dea-27f1-476b-9ef7-d3e2892b411c","remote_addr":"[fd00:1122:3344:101::b]:38882","local_addr":"[::]:4676","unit":"api-server","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.856788747Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"DELETE","req_id":"33d2d7c2-41e4-4d91-9d9f-3adb5769d00c","remote_addr":"[fd00:1122:3344:101::b]:38882","local_addr":"[::]:4676","unit":"api-server","latency_us":745,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.857051593Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"PUT","req_id":"122c63c8-9f82-4307-91ef-707a1d02845f","remote_addr":"[fd00:1122:3344:101::b]:38882","local_addr":"[::]:4676","unit":"api-server","latency_us":127,"response_code":"204"} {"msg":"withdrew tunnel endpoints: [\n TunnelOrigin {\n boundary_addr: fd5c:3d:9074:1d2c::1,\n metric: 18446744073709551615,\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n vni: 99,\n },\n]","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:52.859746025Z","hostname":"oxz_switch","pid":2320} Dec 19 20:29:53.629 INFO request completed, latency_us: 401, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 15dfd20f-5fd6-4384-9ab5-b5164332aa2d, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.002444656Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:62191"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.002508178Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"1732f81f-bc26-49e8-b48b-43f1ad659ac0","remote_addr":"[fd00:1122:3344:101::c]:62191","local_addr":"[::]:4676","unit":"api-server","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.166758287Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:61201"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.166823979Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"GET","req_id":"77f35b91-8805-4c02-80db-291b5020978b","remote_addr":"[fd00:1122:3344:101::c]:61201","local_addr":"[::]:4676","unit":"api-server","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.169774687Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"DELETE","req_id":"b5c20ace-632a-4482-a9ec-abfcfd6af9eb","remote_addr":"[fd00:1122:3344:101::c]:61201","local_addr":"[::]:4676","unit":"api-server","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.169956371Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"PUT","req_id":"b416b948-d924-4abd-94b9-254f19778389","remote_addr":"[fd00:1122:3344:101::c]:61201","local_addr":"[::]:4676","unit":"api-server","latency_us":52,"response_code":"204"} {"msg":"withdrew tunnel endpoints: [\n TunnelOrigin {\n boundary_addr: fd5c:3d:9074:1d2c::1,\n metric: 18446744073709551615,\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n vni: 99,\n },\n]","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.170378151Z","hostname":"oxz_switch","pid":2320} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.275392733Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:35906"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.275467755Z","hostname":"oxz_switch","pid":2320,"uri":"/bfd/peers","method":"GET","req_id":"7ce18a99-f5ce-49e9-9594-37def0ffa613","remote_addr":"[fd00:1122:3344:101::a]:35906","local_addr":"[::]:4676","unit":"api-server","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.373772631Z","hostname":"oxz_switch","pid":2320,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:58422"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.373853293Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"GET","req_id":"dc01d5a0-b86b-4015-adea-c101dba17d0e","remote_addr":"[fd00:1122:3344:101::a]:58422","local_addr":"[::]:4676","unit":"api-server","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.381338117Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"DELETE","req_id":"667969b4-5e19-4a90-a8c6-0f99346487ea","remote_addr":"[fd00:1122:3344:101::a]:58422","local_addr":"[::]:4676","unit":"api-server","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.381538042Z","hostname":"oxz_switch","pid":2320,"uri":"/static/route4","method":"PUT","req_id":"425f7d59-5ff5-4a01-9d01-8a71bf411190","remote_addr":"[fd00:1122:3344:101::a]:58422","local_addr":"[::]:4676","unit":"api-server","latency_us":53,"response_code":"204"} {"msg":"withdrew tunnel endpoints: [\n TunnelOrigin {\n boundary_addr: fd5c:3d:9074:1d2c::1,\n metric: 18446744073709551615,\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n vni: 99,\n },\n]","v":0,"name":"slog-rs","level":30,"time":"2024-12-19T20:29:54.381951311Z","hostname":"oxz_switch","pid":2320} Dec 19 20:29:54.630 INFO request completed, latency_us: 338, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 7487e367-3f31-4eb8-92c6-a515fb1154e3, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:55.629 INFO request completed, latency_us: 302, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 4d21c618-e4f7-46ff-9eae-ffc2203ebad6, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:56.630 INFO request completed, latency_us: 350, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 3581889a-a14b-47e1-8727-c0f8496c9fa2, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:57.633 INFO request completed, latency_us: 352, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 93483750-93ac-46eb-8a58-f0c4758d7239, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:58.630 INFO request completed, latency_us: 394, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 7f33a4e6-debb-4441-8f82-6be805f76648, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:29:59.630 INFO request completed, latency_us: 308, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 6df2a255-3439-49b5-a05c-a069c1ac1ec9, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:30:00.630 INFO request completed, latency_us: 334, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 0fe0396e-e98d-477a-a73f-5260a5868523, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:30:01.629 INFO request completed, latency_us: 338, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 0428c491-6e7c-4801-a45b-e81169e1c788, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 19 20:30:02.633 INFO request completed, latency_us: 342, response_code: 200, uri: /a49cc98a-21bf-4a6e-b607-d8c90c5416e4, method: GET, req_id: 84528a50-a67a-4166-8052-5a5d26ccbe6a, remote_addr: [fd00:1122:3344:101::d]:34694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950