[ Dec 6 20:40:27 Disabled. ] [ Dec 6 20:40:27 Rereading configuration. ] [ Dec 6 20:40:28 Enabled. ] [ Dec 6 20:40:28 Rereading configuration. ] [ Dec 6 20:40:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Dec 6 20:40:28 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.674165735Z","hostname":"oxz_switch","pid":2292} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.70462731Z","hostname":"oxz_switch","pid":2292} {"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.704816555Z","hostname":"oxz_switch","pid":2292} {"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.704849506Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2024-12-06T20:40:28.705287667Z","hostname":"oxz_switch","pid":2292} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.708949941Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709009963Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709024743Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers/{addr}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709043724Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/clear/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709057294Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709073684Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709086555Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709116326Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709130196Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709148186Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709161647Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709175127Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709193538Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709207268Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709226619Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709240129Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709260449Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.70927916Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.70929661Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709310801Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709336191Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709350272Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/routers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709364702Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709383843Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709398833Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709416504Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709430234Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/message-history","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709448314Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/omicron/apply","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709461805Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/exported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709489285Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/imported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709503236Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709518666Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/selected","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709535187Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709543297Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.709555547Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"PUT"} {"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.709563987Z","hostname":"oxz_switch","pid":2292} {"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:28.709569957Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2024-12-06T20:40:28.711056046Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:28.778962346Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:29.780734932Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:30.781304967Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:31.78182214Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:32.782352284Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:33.782872397Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:34.783448432Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:35.783912364Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:36.784472028Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:37.785028122Z","hostname":"oxz_switch","pid":2292} {"msg":"failed to ensure TEP address fd0e:c423:c808:4670::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-06T20:40:38.785576027Z","hostname":"oxz_switch","pid":2292} [ Dec 6 20:40:39 Rereading configuration. ] [ Dec 6 20:40:39 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ] + CONTRACT='' + getopts c: o + CONTRACT=214 + getopts c: o + [[ -n 214 ]] + pgrep -c 214 + PARENT_PID=2291 + [[ -n 2291 ]] + pgrep -P 2291 + CHILD_PID=2292 + [[ -n 2292 ]] + kill -USR1 2292 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:39.961857551Z","hostname":"oxz_switch","pid":2292} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:39.963297188Z","hostname":"oxz_switch","pid":2292} [ Dec 6 20:40:39 Method "refresh" exited with status 0. ] Dec 06 20:40:39.964 DEBG registered DTrace probes Dec 06 20:40:39.964 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Dec 06 20:40:39.964 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197 Dec 06 20:40:39.964 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Dec 06 20:40:39.965 DEBG Nexus IP not provided, will use DNS to resolve it Dec 06 20:40:39.965 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/internal-dns/src/resolver.rs:72 Dec 06 20:40:39.965 DEBG starting producer registration task Dec 06 20:40:39.965 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 15be57ab-19b8-4eda-b168-9f8da913911f, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:283 Dec 06 20:40:39.965 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:40.157520875Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:60242"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:40.158416228Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"PUT","req_id":"e65f3802-17c5-44d8-ac7e-daea6d142ed2","remote_addr":"[fd00:1122:3344:101::1]:60242","local_addr":"[::]:4676","unit":"api-server","latency_us":575,"response_code":"204"} {"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2024-12-06T20:40:40.159073285Z","hostname":"oxz_switch","pid":2292} {"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:40:40.159098556Z","hostname":"oxz_switch","pid":2292} Dec 06 20:40:49.970 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3965e738-3305-4fcd-9da0-4c6e0ce7edc1.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("e9fe68b0-d53c-4161-a621-681677d5b0d6.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd12fb4f-e62d-4cab-a4bd-2024f2546d3b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("fd12fb4f-e62d-4cab-a4bd-2024f2546d3b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::a))) }], valid_until: Instant { tv_sec: 5980, tv_nsec: 119565664 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Dec 06 20:40:49.971 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::b]:12221, component: producer-registration-task Dec 06 20:40:50.003 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:41:05.136 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:41:05.137 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 274.480734ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:41:05.441 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:41:20.402 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:41:20.403 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 656.1236ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:41:21.088 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:41:36.049 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:41:36.050 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 1.472519504s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:41:37.550 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:41:52.510 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:41:52.511 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 1.616971093s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:41:54.154 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:42:09.114 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:42:09.115 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 5.887842414s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:42:15.019 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:42:29.978 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:42:29.979 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 7.81496033s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:42:37.805 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:42:52.764 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:42:52.765 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 11.86049662s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:43:04.652 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:43:19.656 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:43:19.657 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 18.424181365s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 Dec 06 20:43:38.140 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:43:53.153 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Dec 06 20:43:53.160 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 56.114329709s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:29.106478631Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:49203"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:29.122496284Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"ed3ca591-9a50-4e3e-a96c-68ddafbbe6d6","remote_addr":"[fd00:1122:3344:101::c]:49203","local_addr":"[::]:4676","unit":"api-server","latency_us":10287,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:29.141316389Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:41579"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:29.141379671Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"13079630-7516-49ee-a143-bc09dd0d43cb","remote_addr":"[fd00:1122:3344:101::a]:41579","local_addr":"[::]:4676","unit":"api-server","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:30.237823758Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:45032"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:30.238440503Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"462464b0-2dca-4644-bad8-35f10ee3abf8","remote_addr":"[fd00:1122:3344:101::b]:45032","local_addr":"[::]:4676","unit":"api-server","latency_us":38,"response_code":"200"} Dec 06 20:44:49.307 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task Dec 06 20:44:49.364 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "82c704c7-f265-44b7-be24-81173ead6b33", "content-length": "41", "date": "Fri, 06 Dec 2024 20:44:49 GMT"} }), component: producer-registration-task Dec 06 20:44:49.365 DEBG registered with nexus successfully, component: producer-registration-task Dec 06 20:44:49.365 DEBG pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task Dec 06 20:44:57.573 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105 Dec 06 20:44:57.588 INFO request completed, latency_us: 14619, response_code: 200, uri: /15be57ab-19b8-4eda-b168-9f8da913911f, method: GET, req_id: a0cabc39-aa77-4a4a-84dc-b46928b090d0, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 06 20:44:58.449 INFO request completed, latency_us: 327, response_code: 200, uri: /15be57ab-19b8-4eda-b168-9f8da913911f, method: GET, req_id: 3eb03937-1c1e-4d4d-b203-04b5300f569e, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.047363581Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:50247"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.047986087Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"64ae06bc-dd8d-49b2-b35f-c0d357edb3c3","remote_addr":"[fd00:1122:3344:101::c]:50247","local_addr":"[::]:4676","unit":"api-server","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.097385671Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:59341"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.097485943Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"2e3b5133-d007-4805-a406-5df47feed56a","remote_addr":"[fd00:1122:3344:101::a]:59341","local_addr":"[::]:4676","unit":"api-server","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.346111253Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:64232"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.346580275Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"GET","req_id":"6974d8a7-5af2-44c0-9cc5-d057a392f14a","remote_addr":"[fd00:1122:3344:101::c]:64232","local_addr":"[::]:4676","unit":"api-server","latency_us":422,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.349954072Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:42367"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.350047635Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"GET","req_id":"561b9e36-3369-4e9d-91b5-ada304c1954b","remote_addr":"[fd00:1122:3344:101::a]:42367","local_addr":"[::]:4676","unit":"api-server","latency_us":19,"response_code":"200"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.352020355Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"DELETE","req_id":"894b7805-d43a-4c14-a7a0-2c2637fe1b83","remote_addr":"[fd00:1122:3344:101::c]:64232","local_addr":"[::]:4676","unit":"api-server","latency_us":1415,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.352328653Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"PUT","req_id":"282ddd66-15f4-45c9-987a-313d064d5721","remote_addr":"[fd00:1122:3344:101::c]:64232","local_addr":"[::]:4676","unit":"api-server","latency_us":170,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.352417285Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"DELETE","req_id":"8871462f-29db-4de8-a9bd-febd6c2121b4","remote_addr":"[fd00:1122:3344:101::a]:42367","local_addr":"[::]:4676","unit":"api-server","latency_us":38,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.35258862Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"PUT","req_id":"9c4b0303-07f7-46b3-a7c2-07601d178b6f","remote_addr":"[fd00:1122:3344:101::a]:42367","local_addr":"[::]:4676","unit":"api-server","latency_us":62,"response_code":"204"} {"msg":"withdrew tunnel endpoints: [\n TunnelOrigin {\n boundary_addr: fd0e:c423:c808:4670::1,\n metric: 18446744073709551615,\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n vni: 99,\n },\n]","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:44:59.358316748Z","hostname":"oxz_switch","pid":2292} Dec 06 20:44:59.449 INFO request completed, latency_us: 327, response_code: 200, uri: /15be57ab-19b8-4eda-b168-9f8da913911f, method: GET, req_id: 49a62e10-bc68-485d-9d93-d1b73672613f, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.249563154Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:43282"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.249617826Z","hostname":"oxz_switch","pid":2292,"uri":"/bfd/peers","method":"GET","req_id":"0ac55886-d71e-458e-b591-33d038df1a95","remote_addr":"[fd00:1122:3344:101::b]:43282","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.424557065Z","hostname":"oxz_switch","pid":2292,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:42292"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.424638398Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"GET","req_id":"c12cc85b-6f08-4fd7-973f-6db10a298934","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[::]:4676","unit":"api-server","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.426984508Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"DELETE","req_id":"e9ce0219-a48d-4ca9-bdfc-56b86d44e0d8","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[::]:4676","unit":"api-server","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.427237265Z","hostname":"oxz_switch","pid":2292,"uri":"/static/route4","method":"PUT","req_id":"3c353bb2-329d-4d98-a65f-f4eebbcf32e1","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[::]:4676","unit":"api-server","latency_us":103,"response_code":"204"} {"msg":"withdrew tunnel endpoints: [\n TunnelOrigin {\n boundary_addr: fd0e:c423:c808:4670::1,\n metric: 18446744073709551615,\n overlay_prefix: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n vni: 99,\n },\n]","v":0,"name":"slog-rs","level":30,"time":"2024-12-06T20:45:00.427748838Z","hostname":"oxz_switch","pid":2292} Dec 06 20:45:00.451 INFO request completed, latency_us: 1042, response_code: 200, uri: /15be57ab-19b8-4eda-b168-9f8da913911f, method: GET, req_id: 0fc20f61-2f5f-46fb-8464-4bc7e1e3acb9, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950 Dec 06 20:45:01.450 INFO request completed, latency_us: 350, response_code: 200, uri: /15be57ab-19b8-4eda-b168-9f8da913911f, method: GET, req_id: 1b1fd6f7-29a2-4b55-aa82-caf8b980d3a6, remote_addr: [fd00:1122:3344:101::d]:59225, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950