Max level shown:
1 [ Nov 13 07:55:19 Disabled. ]
2 [ Nov 13 07:55:19 Rereading configuration. ]
3 [ Nov 13 07:55:20 Enabled. ]
4 [ Nov 13 07:55:20 Rereading configuration. ]
5 [ Nov 13 07:55:20 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
6 [ Nov 13 07:55:20 Method "start" exited with status 0. ]
72025-11-13T07:55:20.873ZINFOslog-rs: signal handler waiting for context
82025-11-13T07:55:20.896ZINFOslog-rs: signal handler waiting got context
92025-11-13T07:55:20.896ZINFOslog-rs: bgp routers: {}
102025-11-13T07:55:20.896ZINFOslog-rs: bfd peers: []
112025-11-13T07:55:20.897ZERROslog-rs: failed to start stats server: underlay address not found
122025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bestpath/config/fanout unit = api-server versions = all
132025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bestpath/config/fanout unit = api-server versions = all
142025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bfd/peers unit = api-server versions = all
152025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bfd/peers unit = api-server versions = all
162025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bfd/peers/{addr} unit = api-server versions = all
172025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/clear/neighbor unit = api-server versions = all
182025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bgp/config/checker unit = api-server versions = all
192025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/checker unit = api-server versions = all
202025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/config/checker unit = api-server versions = all
212025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bgp/config/checker unit = api-server versions = all
222025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bgp/config/neighbor unit = api-server versions = all
232025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/neighbor unit = api-server versions = all
242025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/config/neighbor unit = api-server versions = all
252025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bgp/config/neighbor unit = api-server versions = all
262025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/neighbors unit = api-server versions = all
272025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bgp/config/origin4 unit = api-server versions = all
282025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/origin4 unit = api-server versions = all
292025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/config/origin4 unit = api-server versions = all
302025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bgp/config/origin4 unit = api-server versions = all
312025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bgp/config/router unit = api-server versions = all
322025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/router unit = api-server versions = all
332025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/config/router unit = api-server versions = all
342025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bgp/config/router unit = api-server versions = all
352025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/routers unit = api-server versions = all
362025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /bgp/config/shaper unit = api-server versions = all
372025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/config/shaper unit = api-server versions = all
382025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/config/shaper unit = api-server versions = all
392025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /bgp/config/shaper unit = api-server versions = all
402025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/message-history unit = api-server versions = all
412025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = POST path = /bgp/omicron/apply unit = api-server versions = all
422025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/status/exported unit = api-server versions = all
432025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/status/imported unit = api-server versions = all
442025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/status/neighbors unit = api-server versions = all
452025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /bgp/status/selected unit = api-server versions = all
462025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = DELETE path = /static/route4 unit = api-server versions = all
472025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = GET path = /static/route4 unit = api-server versions = all
482025-11-13T07:55:20.898ZDEBGslog-rs: registered endpoint local_addr = [::]:4676 method = PUT path = /static/route4 unit = api-server versions = all
492025-11-13T07:55:20.898ZINFOslog-rs: admin: listening on [::]:4676
502025-11-13T07:55:20.898ZINFOslog-rs: listening local_addr = [::]:4676 unit = api-server
512025-11-13T07:55:20.898ZDEBGslog-rs: successfully registered DTrace USDT probes local_addr = [::]:4676 unit = api-server
522025-11-13T07:55:20.898ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
532025-11-13T07:55:21.899ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
542025-11-13T07:55:22.899ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
552025-11-13T07:55:23.900ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
562025-11-13T07:55:24.900ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
572025-11-13T07:55:25.900ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
582025-11-13T07:55:26.901ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
592025-11-13T07:55:27.902ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
602025-11-13T07:55:28.902ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
612025-11-13T07:55:29.902ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
622025-11-13T07:55:30.903ZWARNslog-rs: failed to ensure TEP address fd53:2e21:b44b:1bdc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
63 [ Nov 13 07:55:32 Rereading configuration. ]
64 [ Nov 13 07:55:32 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 215"). ]
65 + CONTRACT=''
66 + getopts c: o
67 + CONTRACT=215
68 + getopts c: o
69 + [[ -n 215 ]]
70 + pgrep -c 215
71 + PARENT_PID=2287
72 + [[ -n 2287 ]]
73 + pgrep -P 2287
74 + CHILD_PID=2288
75 + [[ -n 2288 ]]
76 + kill -USR1 2288
772025-11-13T07:55:32.271ZINFOslog-rs: handling smf refresh
78 [ Nov 13 07:55:32 Method "refresh" exited with status 0. ]
792025-11-13T07:55:32.272ZINFOslog-rs: starting stats server on smf refresh
80 Nov 13 07:55:32.273 DEBG registered DTrace probes
81 Nov 13 07:55:32.273 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
82 Nov 13 07:55:32.274 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328
83 Nov 13 07:55:32.274 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
84 Nov 13 07:55:32.274 DEBG Nexus IP not provided, will use DNS to resolve it
85 Nov 13 07:55:32.274 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/internal-dns/resolver/src/resolver.rs:144
86 Nov 13 07:55:32.274 DEBG starting producer registration task
87 Nov 13 07:55:32.274 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 28426ce3-9b7a-4617-8051-5027e32bfea1, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:279
88 Nov 13 07:55:32.274 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
892025-11-13T07:55:32.569ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::1]:48591 unit = api-server
902025-11-13T07:55:32.570ZINFOslog-rs: request completed latency_us = 502 local_addr = [::]:4676 method = PUT remote_addr = [fd00:1122:3344:101::1]:48591 req_id = 969899d3-9177-44f6-9620-c04b865360da response_code = 204 unit = api-server uri = /static/route4
912025-11-13T07:55:32.571ZERROslog-rs: handling change failed: libnet error route io error No such process (os error 3)
922025-11-13T07:55:32.571ZINFOslog-rs: restarting sync loop
93 Nov 13 07:55:37.277 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5745, tv_nsec: 993350753 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
94 Nov 13 07:55:37.278 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
95 Nov 13 07:55:37.278 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
96 Nov 13 07:55:53.103 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
97 Nov 13 07:55:53.104 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 20.005430592s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
98 Nov 13 07:55:53.231 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5761, tv_nsec: 123654332 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
99 Nov 13 07:55:53.231 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
100 Nov 13 07:55:53.231 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
101 Nov 13 07:56:08.218 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
102 Nov 13 07:56:08.222 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 35.163568198s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
103 Nov 13 07:56:08.803 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5776, tv_nsec: 736702001 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
104 Nov 13 07:56:08.803 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
105 Nov 13 07:56:08.807 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
106 Nov 13 07:56:23.771 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
107 Nov 13 07:56:23.773 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 50.756141375s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
108 Nov 13 07:56:24.931 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5792, tv_nsec: 909493405 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
109 Nov 13 07:56:24.931 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
110 Nov 13 07:56:24.933 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
111 Nov 13 07:56:39.893 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
112 Nov 13 07:56:39.893 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 66.922089694s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
113 Nov 13 07:56:42.366 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5810, tv_nsec: 392589459 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
114 Nov 13 07:56:42.367 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
115 Nov 13 07:56:42.367 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
116 Nov 13 07:56:57.327 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
117 Nov 13 07:56:57.327 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 84.403643518s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
118 Nov 13 07:57:01.084 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5829, tv_nsec: 161023612 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
119 Nov 13 07:57:01.084 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
120 Nov 13 07:57:01.085 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
121 Nov 13 07:57:16.045 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
122 Nov 13 07:57:16.045 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 103.172241693s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
123 Nov 13 07:57:27.188 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5855, tv_nsec: 335956430 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
124 Nov 13 07:57:27.189 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
125 Nov 13 07:57:27.189 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
126 Nov 13 07:57:42.149 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
127 Nov 13 07:57:42.149 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 129.346526801s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
128 Nov 13 07:57:54.650 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5882, tv_nsec: 872637371 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
129 Nov 13 07:57:54.651 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
130 Nov 13 07:57:54.651 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
131 Nov 13 07:58:09.611 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
132 Nov 13 07:58:09.611 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 156.883692992s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
133 Nov 13 07:58:42.511 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6564a4a0-ff58-4081-9860-74f24e488435.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("79428d63-073a-4ba3-9687-ed29ef6cec3a.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("bcb58129-79cc-406c-aa16-455a5cd9f031.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 5930, tv_nsec: 863304962 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
134 Nov 13 07:58:42.512 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task
135 Nov 13 07:58:42.513 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task
136 Nov 13 07:58:57.514 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::b]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
137 Nov 13 07:58:57.514 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)", delay: 204.876167427s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
1382025-11-13T07:59:02.757ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::b]:55544 unit = api-server
1392025-11-13T07:59:02.770ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::c]:50384 unit = api-server
1402025-11-13T07:59:02.783ZINFOslog-rs: request completed latency_us = 18904 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::b]:55544 req_id = bc9830ae-f22d-4dcf-92bb-23d46db8f218 response_code = 200 unit = api-server uri = /bfd/peers
1412025-11-13T07:59:02.783ZINFOslog-rs: request completed latency_us = 6949 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::c]:50384 req_id = b3f0c316-838a-4fd0-9901-2cb034344234 response_code = 200 unit = api-server uri = /bfd/peers
1422025-11-13T07:59:02.830ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::a]:49846 unit = api-server
1432025-11-13T07:59:02.830ZINFOslog-rs: request completed latency_us = 58 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::a]:49846 req_id = cb3eff95-5174-42a8-9816-26839c6c6862 response_code = 200 unit = api-server uri = /bfd/peers
1442025-11-13T07:59:32.749ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::c]:62015 unit = api-server
1452025-11-13T07:59:32.749ZINFOslog-rs: request completed latency_us = 49 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::c]:62015 req_id = de9c6628-c7cf-4c1e-8277-1e186fef6084 response_code = 200 unit = api-server uri = /bfd/peers
1462025-11-13T07:59:32.850ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::a]:60949 unit = api-server
1472025-11-13T07:59:32.850ZINFOslog-rs: request completed latency_us = 51 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::a]:60949 req_id = 7c84630d-d63a-4065-8492-f011a9a7c457 response_code = 200 unit = api-server uri = /bfd/peers
1482025-11-13T07:59:32.964ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::b]:60775 unit = api-server
1492025-11-13T07:59:32.964ZINFOslog-rs: request completed latency_us = 40 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::b]:60775 req_id = aae24613-9248-4b09-9539-d9638d16379f response_code = 200 unit = api-server uri = /bfd/peers
1502025-11-13T07:59:33.107ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::c]:60780 unit = api-server
1512025-11-13T07:59:33.117ZINFOslog-rs: request completed latency_us = 9966 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::c]:60780 req_id = d25782b2-8eb8-43bb-8978-c8916001e6dc response_code = 200 unit = api-server uri = /static/route4
1522025-11-13T07:59:33.121ZINFOslog-rs: request completed latency_us = 3375 local_addr = [::]:4676 method = DELETE remote_addr = [fd00:1122:3344:101::c]:60780 req_id = 48da6bd7-e169-4ee3-a554-f2e6c8caded5 response_code = 204 unit = api-server uri = /static/route4
1532025-11-13T07:59:33.121ZINFOslog-rs: request completed latency_us = 135 local_addr = [::]:4676 method = PUT remote_addr = [fd00:1122:3344:101::c]:60780 req_id = 6004d555-478d-4236-a7e2-f57f99e78df2 response_code = 204 unit = api-server uri = /static/route4
1542025-11-13T07:59:33.128ZINFOslog-rs: withdrew tunnel endpoints: [ TunnelOrigin { boundary_addr: fd53:2e21:b44b:1bdc::1, metric: 18446744073709551615, overlay_prefix: V4( Ipv4Net { addr: 0.0.0.0, width: 0, }, ), vni: 99, }, ]
1552025-11-13T07:59:33.203ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::b]:44619 unit = api-server
1562025-11-13T07:59:33.204ZINFOslog-rs: request completed latency_us = 71 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::b]:44619 req_id = 14da819d-e4c8-440d-99cc-87f0c683bde8 response_code = 200 unit = api-server uri = /static/route4
1572025-11-13T07:59:33.204ZINFOslog-rs: request completed latency_us = 126 local_addr = [::]:4676 method = DELETE remote_addr = [fd00:1122:3344:101::b]:44619 req_id = 828f6cf0-aef0-4aa7-9c8b-ec53567352f9 response_code = 204 unit = api-server uri = /static/route4
1582025-11-13T07:59:33.204ZINFOslog-rs: request completed latency_us = 95 local_addr = [::]:4676 method = PUT remote_addr = [fd00:1122:3344:101::b]:44619 req_id = 393394b3-5e01-47af-9fb1-92ffd7b460ee response_code = 204 unit = api-server uri = /static/route4
1592025-11-13T07:59:33.206ZINFOslog-rs: withdrew tunnel endpoints: [ TunnelOrigin { boundary_addr: fd53:2e21:b44b:1bdc::1, metric: 18446744073709551615, overlay_prefix: V4( Ipv4Net { addr: 0.0.0.0, width: 0, }, ), vni: 99, }, ]
1602025-11-13T07:59:33.275ZINFOslog-rs: accepted connection local_addr = [::]:4676 remote_addr = [fd00:1122:3344:101::a]:51711 unit = api-server
1612025-11-13T07:59:33.275ZINFOslog-rs: request completed latency_us = 64 local_addr = [::]:4676 method = GET remote_addr = [fd00:1122:3344:101::a]:51711 req_id = 4f3f8d3c-88bd-4969-a92e-b8d8296770e3 response_code = 200 unit = api-server uri = /static/route4
1622025-11-13T07:59:33.275ZINFOslog-rs: request completed latency_us = 149 local_addr = [::]:4676 method = DELETE remote_addr = [fd00:1122:3344:101::a]:51711 req_id = f9ac424a-952c-47bf-8ba7-5826ebd15945 response_code = 204 unit = api-server uri = /static/route4
1632025-11-13T07:59:33.275ZINFOslog-rs: request completed latency_us = 83 local_addr = [::]:4676 method = PUT remote_addr = [fd00:1122:3344:101::a]:51711 req_id = 01bcc607-0e71-44fa-b706-f6ca8b748a75 response_code = 204 unit = api-server uri = /static/route4
1642025-11-13T07:59:33.277ZINFOslog-rs: withdrew tunnel endpoints: [ TunnelOrigin { boundary_addr: fd53:2e21:b44b:1bdc::1, metric: 18446744073709551615, overlay_prefix: V4( Ipv4Net { addr: 0.0.0.0, width: 0, }, ), vni: 99, }, ]