[ Nov 18 15:12:59 Disabled. ]
[ Nov 18 15:12:59 Rereading configuration. ]
[ Nov 18 15:13:00 Enabled. ]
[ Nov 18 15:13:00 Rereading configuration. ]
[ Nov 18 15:13:00 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Nov 18 15:13:00 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.65998405Z","hostname":"oxz_switch","pid":2283}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.681304236Z","hostname":"oxz_switch","pid":2283}
{"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.681439031Z","hostname":"oxz_switch","pid":2283}
{"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.681520923Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-11-18T15:13:00.682171614Z","hostname":"oxz_switch","pid":2283}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683929221Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683944691Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683961462Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683969452Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683978922Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683986063Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/clear/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.683993033Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684015574Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684023234Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684033294Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684040414Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684049525Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684056945Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684066995Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684074385Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684094576Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684102466Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684124137Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684132217Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684146948Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684153758Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684164358Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684171239Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684181059Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684188059Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684197369Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.6842045Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.6842214Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.68422829Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684237891Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684245161Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/exported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684252071Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684261862Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684269272Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684278042Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684285082Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684293212Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"PUT"}
{"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.684300453Z","hostname":"oxz_switch","pid":2283}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:00.684308393Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2025-11-18T15:13:00.684314443Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:00.684698456Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:01.685189928Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:02.685632608Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:03.686033328Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:04.686425156Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:05.686714411Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:06.687192954Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:07.687551081Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:08.687854728Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:09.688156024Z","hostname":"oxz_switch","pid":2283}
{"msg":"failed to ensure TEP address fd3d:8055:290b:b2ee::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-11-18T15:13:10.688456849Z","hostname":"oxz_switch","pid":2283}
[ Nov 18 15:13:11 Rereading configuration. ]
[ Nov 18 15:13:11 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=214
+ getopts c: o
+ [[ -n 214 ]]
+ pgrep -c 214
+ PARENT_PID=2282
+ [[ -n 2282 ]]
+ pgrep -P 2282
+ CHILD_PID=2283
+ [[ -n 2283 ]]
+ kill -USR1 2283
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:11.99537012Z","hostname":"oxz_switch","pid":2283}
[ Nov 18 15:13:11 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:11.997341323Z","hostname":"oxz_switch","pid":2283}
Nov 18 15:13:11.998 DEBG registered DTrace probes
Nov 18 15:13:11.998 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Nov 18 15:13:11.998 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328
Nov 18 15:13:11.998 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Nov 18 15:13:11.998 DEBG Nexus IP not provided, will use DNS to resolve it
Nov 18 15:13:11.998 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/internal-dns/resolver/src/resolver.rs:144
Nov 18 15:13:11.999 DEBG starting producer registration task
Nov 18 15:13:11.999 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 3bccc802-0d39-4c65-8ed6-73b1a76237cf, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:279
Nov 18 15:13:11.999 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:12.292838062Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:37732"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:12.293607967Z","hostname":"oxz_switch","pid":2283,"uri":"/static/route4","method":"PUT","req_id":"5c5b74b3-9d81-4083-8b1b-553b09654a8f","remote_addr":"[fd00:1122:3344:101::1]:37732","local_addr":"[::]:4676","unit":"api-server","latency_us":550,"response_code":204}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2025-11-18T15:13:12.294761693Z","hostname":"oxz_switch","pid":2283}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:13:12.294803995Z","hostname":"oxz_switch","pid":2283}
Nov 18 15:13:17.002 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3793, tv_nsec: 307333069 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:13:17.003 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:13:17.003 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:13:32.171 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Nov 18 15:13:32.171 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 20.005034276s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Nov 18 15:13:32.533 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3808, tv_nsec: 671130258 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:13:32.533 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:13:32.534 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:13:48.545 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Nov 18 15:13:48.560 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 36.352553018s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Nov 18 15:13:51.609 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3827, tv_nsec: 690475648 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:13:51.612 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:13:51.615 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:14:06.661 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Nov 18 15:14:06.673 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 54.436665837s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Nov 18 15:14:08.026 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3844, tv_nsec: 97294569 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:14:08.501 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:14:08.504 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:14:23.506 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Nov 18 15:14:23.506 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 71.271964635s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Nov 18 15:14:26.292 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3862, tv_nsec: 362684841 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:14:26.294 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:14:26.294 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:14:41.295 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Nov 18 15:14:41.295 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 89.061468014s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:43.690577089Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:43960"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:43.71049011Z","hostname":"oxz_switch","pid":2283,"uri":"/bfd/peers","method":"GET","req_id":"0c81402b-e9fa-491f-9cf3-c672a3923d7b","remote_addr":"[fd00:1122:3344:101::c]:43960","local_addr":"[::]:4676","unit":"api-server","latency_us":13895,"response_code":200}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:43.799768723Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:42111"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:43.800087403Z","hostname":"oxz_switch","pid":2283,"uri":"/bfd/peers","method":"GET","req_id":"6e762598-378c-4176-9680-356c3110e439","remote_addr":"[fd00:1122:3344:101::b]:42111","local_addr":"[::]:4676","unit":"api-server","latency_us":51,"response_code":200}
Nov 18 15:14:44.174 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("3bd16592-14ca-4d00-959b-9efb0f600b0f.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("6b993aa7-f2e0-4e9a-b1d1-ea7a8ea320d6.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("ca18d5c5-ecce-41d0-8dd0-07e6b45d67ac.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 3880, tv_nsec: 244129110 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Nov 18 15:14:44.175 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Nov 18 15:14:44.175 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Nov 18 15:14:44.250 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "f844134f-cc93-4834-a003-dd7b68f82a40", "content-length": "41", "date": "Tue, 18 Nov 2025 15:14:44 GMT"} }), component: producer-registration-task
Nov 18 15:14:44.250 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:45.05619709Z","hostname":"oxz_switch","pid":2283,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:44342"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-11-18T15:14:45.056335774Z","hostname":"oxz_switch","pid":2283,"uri":"/bfd/peers","method":"GET","req_id":"98b05441-288d-4b91-a78b-9ac7d60301da","remote_addr":"[fd00:1122:3344:101::a]:44342","local_addr":"[::]:4676","unit":"api-server","latency_us":62,"response_code":200}
Nov 18 15:14:57.636 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:1025
Nov 18 15:14:57.649 INFO request completed, latency_us: 12699, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: a9ca52d3-a396-4c73-8fd3-d6aff2956f86, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Nov 18 15:14:58.335 INFO request completed, latency_us: 431, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: 1eeb0ad6-da11-47ab-8acf-e7999eb1e405, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Nov 18 15:14:59.335 INFO request completed, latency_us: 357, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: ccff2fa5-fcc7-43ae-bf2e-ebbf43129e9f, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Nov 18 15:15:00.335 INFO request completed, latency_us: 426, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: ada98faf-41d7-443f-bbd6-f4701ede1e9b, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Nov 18 15:15:01.334 INFO request completed, latency_us: 383, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: eab9443c-238a-4a94-8c1c-42af8b665d09, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Nov 18 15:15:02.335 INFO request completed, latency_us: 452, response_code: 200, uri: /3bccc802-0d39-4c65-8ed6-73b1a76237cf, method: GET, req_id: f26bf6c7-2c15-44be-91e8-d754ad788af1, remote_addr: [fd00:1122:3344:101::d]:49103, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867