1 | | [ Jul 18 19:40:39 Disabled. ] |
2 | | [ Jul 18 19:40:39 Rereading configuration. ] |
3 | | [ Jul 18 19:40:40 Enabled. ] |
4 | | [ Jul 18 19:40:40 Rereading configuration. ] |
5 | | [ Jul 18 19:40:40 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] |
6 | | [ Jul 18 19:40:40 Method "start" exited with status 0. ] |
7 | 2025-07-18T19:40:40.498Z | INFO | slog-rs: signal handler waiting for context
|
8 | 2025-07-18T19:40:40.519Z | INFO | slog-rs: signal handler waiting got context
|
9 | 2025-07-18T19:40:40.519Z | INFO | slog-rs: bgp routers: {}
|
10 | 2025-07-18T19:40:40.519Z | INFO | slog-rs: bfd peers: []
|
11 | 2025-07-18T19:40:40.520Z | ERRO | slog-rs: failed to start stats server: underlay address not found
|
12 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bfd/peers
unit = api-server
versions = all
|
13 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bfd/peers
unit = api-server
versions = all
|
14 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bfd/peers/{addr}
unit = api-server
versions = all
|
15 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/clear/neighbor
unit = api-server
versions = all
|
16 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bgp/config/checker
unit = api-server
versions = all
|
17 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/checker
unit = api-server
versions = all
|
18 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/config/checker
unit = api-server
versions = all
|
19 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bgp/config/checker
unit = api-server
versions = all
|
20 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bgp/config/neighbor
unit = api-server
versions = all
|
21 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/neighbor
unit = api-server
versions = all
|
22 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/config/neighbor
unit = api-server
versions = all
|
23 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bgp/config/neighbor
unit = api-server
versions = all
|
24 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/neighbors
unit = api-server
versions = all
|
25 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bgp/config/origin4
unit = api-server
versions = all
|
26 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/origin4
unit = api-server
versions = all
|
27 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/config/origin4
unit = api-server
versions = all
|
28 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bgp/config/origin4
unit = api-server
versions = all
|
29 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bgp/config/router
unit = api-server
versions = all
|
30 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/router
unit = api-server
versions = all
|
31 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/config/router
unit = api-server
versions = all
|
32 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bgp/config/router
unit = api-server
versions = all
|
33 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/routers
unit = api-server
versions = all
|
34 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /bgp/config/shaper
unit = api-server
versions = all
|
35 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/config/shaper
unit = api-server
versions = all
|
36 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/config/shaper
unit = api-server
versions = all
|
37 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /bgp/config/shaper
unit = api-server
versions = all
|
38 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/message-history
unit = api-server
versions = all
|
39 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = POST
path = /bgp/omicron/apply
unit = api-server
versions = all
|
40 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/status/exported
unit = api-server
versions = all
|
41 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/status/imported
unit = api-server
versions = all
|
42 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/status/neighbors
unit = api-server
versions = all
|
43 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /bgp/status/selected
unit = api-server
versions = all
|
44 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = DELETE
path = /static/route4
unit = api-server
versions = all
|
45 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = GET
path = /static/route4
unit = api-server
versions = all
|
46 | 2025-07-18T19:40:40.521Z | DEBG | slog-rs: registered endpoint
local_addr = [::]:4676
method = PUT
path = /static/route4
unit = api-server
versions = all
|
47 | 2025-07-18T19:40:40.521Z | INFO | slog-rs: admin: listening on [::]:4676
|
48 | 2025-07-18T19:40:40.521Z | INFO | slog-rs: listening
local_addr = [::]:4676
unit = api-server
|
49 | 2025-07-18T19:40:40.522Z | DEBG | slog-rs: successfully registered DTrace USDT probes
local_addr = [::]:4676
unit = api-server
|
50 | 2025-07-18T19:40:40.522Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
51 | 2025-07-18T19:40:41.523Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
52 | 2025-07-18T19:40:42.523Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
53 | 2025-07-18T19:40:43.523Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
54 | 2025-07-18T19:40:44.524Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
55 | 2025-07-18T19:40:45.524Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
56 | 2025-07-18T19:40:46.525Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
57 | 2025-07-18T19:40:47.525Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
58 | 2025-07-18T19:40:48.525Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
59 | 2025-07-18T19:40:49.526Z | WARN | slog-rs: failed to ensure TEP address fd85:76fc:6cfc:8bc8::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)
|
60 | | [ Jul 18 19:40:50 Rereading configuration. ] |
61 | | [ Jul 18 19:40:50 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ] |
62 | | + CONTRACT='' |
63 | | + getopts c: o |
64 | | + CONTRACT=214 |
65 | | + getopts c: o |
66 | | + [[ -n 214 ]] |
67 | | + pgrep -c 214 |
68 | | + PARENT_PID=2278 |
69 | | + [[ -n 2278 ]] |
70 | | + pgrep -P 2278 |
71 | | + CHILD_PID=2279 |
72 | | + [[ -n 2279 ]] |
73 | | + kill -USR1 2279 |
74 | 2025-07-18T19:40:50.904Z | INFO | slog-rs: handling smf refresh
|
75 | 2025-07-18T19:40:50.905Z | INFO | slog-rs: starting stats server on smf refresh
|
76 | | [ Jul 18 19:40:50 Method "refresh" exited with status 0. ] |
77 | | Jul 18 19:40:50.906 DEBG registered DTrace probes |
78 | | Jul 18 19:40:50.906 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot |
79 | | Jul 18 19:40:50.906 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328 |
80 | | Jul 18 19:40:50.907 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot |
81 | | Jul 18 19:40:50.907 DEBG Nexus IP not provided, will use DNS to resolve it |
82 | | Jul 18 19:40:50.907 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/internal-dns/resolver/src/resolver.rs:111 |
83 | | Jul 18 19:40:50.907 DEBG starting producer registration task |
84 | | Jul 18 19:40:50.907 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 2ab58cea-1973-4965-8759-ba7489e0fe93, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:279 |
85 | | Jul 18 19:40:50.907 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task |
86 | 2025-07-18T19:40:51.203Z | INFO | slog-rs: accepted connection
local_addr = [::]:4676
remote_addr = [fd00:1122:3344:101::1]:64102
unit = api-server
|
87 | 2025-07-18T19:40:51.204Z | INFO | slog-rs: request completed
latency_us = 449
local_addr = [::]:4676
method = PUT
remote_addr = [fd00:1122:3344:101::1]:64102
req_id = 46251a79-cea6-42de-9116-5be9098cb359
response_code = 204
unit = api-server
uri = /static/route4
|
88 | 2025-07-18T19:40:51.205Z | ERRO | slog-rs: handling change failed: libnet error route io error No such process (os error 3)
|
89 | 2025-07-18T19:40:51.205Z | INFO | slog-rs: restarting sync loop
|
90 | | Jul 18 19:40:55.909 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 632, tv_nsec: 475563848 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
91 | | Jul 18 19:40:55.910 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
92 | | Jul 18 19:40:55.910 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
93 | | Jul 18 19:41:10.268 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
94 | | Jul 18 19:41:10.269 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 20.003707608s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
95 | | Jul 18 19:41:10.443 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 647, tv_nsec: 651780325 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
96 | | Jul 18 19:41:10.444 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
97 | | Jul 18 19:41:10.444 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
98 | | Jul 18 19:41:25.484 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
99 | | Jul 18 19:41:25.484 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 35.180095647s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
100 | | Jul 18 19:41:26.067 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 663, tv_nsec: 234048576 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
101 | | Jul 18 19:41:26.067 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
102 | | Jul 18 19:41:26.067 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
103 | | Jul 18 19:41:41.110 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
104 | | Jul 18 19:41:41.110 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 50.763219334s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
105 | | Jul 18 19:41:42.244 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 679, tv_nsec: 367958248 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
106 | | Jul 18 19:41:42.245 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
107 | | Jul 18 19:41:42.245 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
108 | | Jul 18 19:41:57.288 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
109 | | Jul 18 19:41:57.288 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 66.897091035s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
110 | | Jul 18 19:42:00.233 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 697, tv_nsec: 306822166 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
111 | | Jul 18 19:42:00.233 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
112 | | Jul 18 19:42:00.233 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
113 | | Jul 18 19:42:15.274 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
114 | | Jul 18 19:42:15.274 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 84.834322239s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
115 | | Jul 18 19:42:17.958 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 714, tv_nsec: 984201178 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
116 | | Jul 18 19:42:17.959 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
117 | | Jul 18 19:42:17.959 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
118 | | Jul 18 19:42:33.001 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
119 | | Jul 18 19:42:33.001 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 102.513201482s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
120 | | Jul 18 19:42:38.684 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 735, tv_nsec: 654265628 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
121 | | Jul 18 19:42:38.685 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
122 | | Jul 18 19:42:38.685 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
123 | | Jul 18 19:42:53.727 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
124 | | Jul 18 19:42:53.727 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 123.182833739s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
125 | | Jul 18 19:43:06.146 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 763, tv_nsec: 41933608 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
126 | | Jul 18 19:43:06.147 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
127 | | Jul 18 19:43:06.147 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
128 | | Jul 18 19:43:21.189 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
129 | | Jul 18 19:43:21.190 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 150.590402688s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
130 | | Jul 18 19:43:50.219 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 807, tv_nsec: 95091176 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
131 | | Jul 18 19:43:50.221 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
132 | | Jul 18 19:43:50.227 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
133 | | Jul 18 19:44:07.214 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task |
134 | | Jul 18 19:44:07.224 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 196.617485819s, count: 8, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/1d3958c/oximeter/producer/src/lib.rs:371 |
135 | 2025-07-18T19:44:33.188Z | INFO | slog-rs: accepted connection
local_addr = [::]:4676
remote_addr = [fd00:1122:3344:101::c]:37720
unit = api-server
|
136 | 2025-07-18T19:44:33.206Z | INFO | slog-rs: request completed
latency_us = 9367
local_addr = [::]:4676
method = GET
remote_addr = [fd00:1122:3344:101::c]:37720
req_id = 9b9675e8-ff43-413f-8c53-edebcafbbf55
response_code = 200
unit = api-server
uri = /bfd/peers
|
137 | 2025-07-18T19:44:33.238Z | INFO | slog-rs: accepted connection
local_addr = [::]:4676
remote_addr = [fd00:1122:3344:101::b]:56381
unit = api-server
|
138 | 2025-07-18T19:44:33.238Z | INFO | slog-rs: request completed
latency_us = 34
local_addr = [::]:4676
method = GET
remote_addr = [fd00:1122:3344:101::b]:56381
req_id = 9b8887ba-fdac-4c28-a89b-0fd5f30e7c84
response_code = 200
unit = api-server
uri = /bfd/peers
|
139 | 2025-07-18T19:44:33.431Z | INFO | slog-rs: accepted connection
local_addr = [::]:4676
remote_addr = [fd00:1122:3344:101::a]:56711
unit = api-server
|
140 | 2025-07-18T19:44:33.431Z | INFO | slog-rs: request completed
latency_us = 39
local_addr = [::]:4676
method = GET
remote_addr = [fd00:1122:3344:101::a]:56711
req_id = 4dc74043-8c69-4155-bc01-e887a2dce182
response_code = 200
unit = api-server
uri = /bfd/peers
|
141 | | Jul 18 19:44:45.328 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("b2568d68-c2f1-4ec5-9f40-43ecf91a898a.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0ab1674-fd55-45ce-baf5-f683df76cfe7.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2e32d3e-2f29-4056-a216-294e9d23f41b.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 862, tv_nsec: 201958755 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver |
142 | | Jul 18 19:44:45.329 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task |
143 | | Jul 18 19:44:45.332 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task |
144 | | Jul 18 19:44:45.404 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "14442baa-70a2-4784-bc83-24be89aa608e", "content-length": "41", "date": "Fri, 18 Jul 2025 19:44:45 GMT"} }), component: producer-registration-task |
145 | | Jul 18 19:44:45.404 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task |
146 | | Jul 18 19:44:47.689 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:1025 |
147 | | Jul 18 19:44:47.700 INFO request completed, latency_us: 10969, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: f4ca3a3d-fdac-4b5f-aaa8-1577b7235199, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
148 | | Jul 18 19:44:48.690 INFO request completed, latency_us: 293, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: f2732dd9-b021-45c2-9e23-304c8c7a12bb, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
149 | | Jul 18 19:44:49.689 INFO request completed, latency_us: 306, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: f99a4fb9-fc32-41f0-a6eb-ee2d9678a27b, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
150 | | Jul 18 19:44:50.690 INFO request completed, latency_us: 466, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 64becfd7-fe2d-4e56-8a76-8a584f2abdb7, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
151 | | Jul 18 19:44:51.690 INFO request completed, latency_us: 305, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 8fc7a279-f370-4765-bf6b-035253ba58cb, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
152 | | Jul 18 19:44:52.843 INFO request completed, latency_us: 431, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 770596ee-8d83-4bdb-a530-97dbe1389152, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
153 | | Jul 18 19:44:53.843 INFO request completed, latency_us: 406, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 74549b50-8af0-4702-bd2b-9651070a7365, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
154 | | Jul 18 19:44:54.843 INFO request completed, latency_us: 359, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: b0197a33-3ffc-4511-aa76-78a4b1b62deb, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
155 | | Jul 18 19:44:55.842 INFO request completed, latency_us: 361, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 3c5f4ce2-0f45-4acc-bb32-2bb1b5fec778, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
156 | | Jul 18 19:44:56.843 INFO request completed, latency_us: 354, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: 49143ad8-ac85-4996-b8d2-30a435b24c0d, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |
157 | | Jul 18 19:44:57.843 INFO request completed, latency_us: 325, response_code: 200, uri: /2ab58cea-1973-4965-8759-ba7489e0fe93, method: GET, req_id: c290d1f2-f41f-404a-b8cf-89a6dcc24f67, remote_addr: [fd00:1122:3344:101::d]:55622, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867 |