[ Dec 18 22:11:04 Disabled. ]
[ Dec 18 22:11:04 Rereading configuration. ]
[ Dec 18 22:11:05 Enabled. ]
[ Dec 18 22:11:05 Rereading configuration. ]
[ Dec 18 22:11:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Dec 18 22:11:05 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.485461054Z","hostname":"oxz_switch","pid":2262}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.5061256Z","hostname":"oxz_switch","pid":2262}
{"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.506308874Z","hostname":"oxz_switch","pid":2262}
{"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.506360525Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2024-12-18T22:11:05.506847094Z","hostname":"oxz_switch","pid":2262}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510412274Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510456205Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510465355Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510477565Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/clear/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510485365Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510494986Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510502196Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510511536Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510519446Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510530266Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510537646Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510545997Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510561187Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510569387Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510581718Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510589878Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510609648Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510617718Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510629468Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510638928Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510654099Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510664009Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510672389Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.51068302Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.51069078Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.51070105Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.51071143Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510732501Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510756411Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/exported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510770791Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510779041Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510787302Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510800352Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510808132Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.510825892Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"PUT"}
{"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.510848113Z","hostname":"oxz_switch","pid":2262}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:05.510855433Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2024-12-18T22:11:05.512308191Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:05.579627992Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:06.58153439Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:07.582175982Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:08.582731003Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:09.583255953Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:10.583801534Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:11.584346565Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:12.584880105Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:13.585407355Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:14.585986177Z","hostname":"oxz_switch","pid":2262}
{"msg":"failed to ensure TEP address fd77:b23e:4e43:dfd3::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2024-12-18T22:11:15.586513407Z","hostname":"oxz_switch","pid":2262}
[ Dec 18 22:11:17 Rereading configuration. ]
[ Dec 18 22:11:17 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=214
+ getopts c: o
+ [[ -n 214 ]]
+ pgrep -c 214
+ PARENT_PID=2261
+ [[ -n 2261 ]]
+ pgrep -P 2261
+ CHILD_PID=2262
+ [[ -n 2262 ]]
+ kill -USR1 2262
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:17.188956927Z","hostname":"oxz_switch","pid":2262}
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:17.190392765Z","hostname":"oxz_switch","pid":2262}
[ Dec 18 22:11:17 Method "refresh" exited with status 0. ]
Dec 18 22:11:17.191 DEBG registered DTrace probes
Dec 18 22:11:17.191 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Dec 18 22:11:17.191 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197
Dec 18 22:11:17.192 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Dec 18 22:11:17.192 DEBG Nexus IP not provided, will use DNS to resolve it
Dec 18 22:11:17.192 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/internal-dns/src/resolver.rs:72
Dec 18 22:11:17.192 DEBG starting producer registration task
Dec 18 22:11:17.192 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: e38ae931-fe39-47a5-a036-196300c6bb27, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:283
Dec 18 22:11:17.192 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:17.367603922Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:51501"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:17.368310166Z","hostname":"oxz_switch","pid":2262,"uri":"/static/route4","method":"PUT","req_id":"08b812f5-865c-4360-80a0-6e3045e52163","remote_addr":"[fd00:1122:3344:101::1]:51501","local_addr":"[::]:4676","unit":"api-server","latency_us":448,"response_code":"204"}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2024-12-18T22:11:17.368787385Z","hostname":"oxz_switch","pid":2262}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:11:17.368801696Z","hostname":"oxz_switch","pid":2262}
Dec 18 22:11:27.195 WARN failed to lookup Nexus IP, will retry, error: "no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }", delay: 344.071604ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:391
Dec 18 22:11:27.541 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("175d3cc8-9a02-4cf3-970c-8ec76a875436.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("9f269ac0-3a79-44fd-9bc5-1f6f5cdfd395.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c0e3d114-075e-41a4-a4df-fa8131bbea97.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c0e3d114-075e-41a4-a4df-fa8131bbea97.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::b))) }], valid_until: Instant { tv_sec: 777, tv_nsec: 646542875 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 18 22:11:27.542 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 18 22:11:27.574 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:11:42.278 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:11:42.279 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 256.490351ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:11:42.565 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:11:57.526 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:11:57.527 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 406.570044ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:11:57.964 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:12:12.925 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:12:12.926 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 916.058189ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:12:13.873 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:12:28.833 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:12:28.834 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 2.811171392s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:12:31.671 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:12:46.631 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:12:46.632 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 4.135602341s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:12:50.788 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:13:05.751 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:13:05.752 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 4.485515158s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:13:10.269 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:13:25.270 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:13:25.271 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 11.350788455s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
Dec 18 22:13:39.791 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:13:54.793 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 18 22:13:54.798 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 35.315395969s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/48125c8/oximeter/producer/src/lib.rs:424
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.011761117Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:33568"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.025335774Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"7faac41a-5313-4b19-a51d-56c1e96d9697","remote_addr":"[fd00:1122:3344:101::a]:33568","local_addr":"[::]:4676","unit":"api-server","latency_us":8930,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.026835893Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:53177"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.027033007Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"020b6831-b286-41b3-8606-bffdd560a16e","remote_addr":"[fd00:1122:3344:101::b]:53177","local_addr":"[::]:4676","unit":"api-server","latency_us":20,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.962746865Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:55514"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:13.962820207Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"f1b8cc9d-456b-4818-bf47-496908091a3e","remote_addr":"[fd00:1122:3344:101::c]:55514","local_addr":"[::]:4676","unit":"api-server","latency_us":27,"response_code":"200"}
Dec 18 22:14:30.151 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 18 22:14:30.208 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "2d1bcb6d-d866-4266-a051-bce4a68aeb60", "content-length": "41", "date": "Wed, 18 Dec 2024 22:14:29 GMT"} }), component: producer-registration-task
Dec 18 22:14:30.210 DEBG registered with nexus successfully, component: producer-registration-task
Dec 18 22:14:30.210 DEBG pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
Dec 18 22:14:42.634 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105
Dec 18 22:14:42.645 INFO request completed, latency_us: 10395, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: c5c9b52c-cd8f-4578-abac-bb7e17d46fb1, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:42.864279136Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:34208"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:42.864330977Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"402b92dc-e0e4-49c0-80f7-8f61dfe1fae2","remote_addr":"[fd00:1122:3344:101::a]:34208","local_addr":"[::]:4676","unit":"api-server","latency_us":25,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:42.932395613Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:57436"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:42.932464944Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"bf9eb50a-ecd5-4615-a85a-09b107a5f4ae","remote_addr":"[fd00:1122:3344:101::b]:57436","local_addr":"[::]:4676","unit":"api-server","latency_us":19,"response_code":"200"}
Dec 18 22:14:43.635 INFO request completed, latency_us: 295, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 5931f5e8-e7bd-4814-b495-d0d927188b37, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:43.907064635Z","hostname":"oxz_switch","pid":2262,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:60884"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-12-18T22:14:43.907126437Z","hostname":"oxz_switch","pid":2262,"uri":"/bfd/peers","method":"GET","req_id":"154616f7-885f-4eba-b5b6-d0548fdb2913","remote_addr":"[fd00:1122:3344:101::c]:60884","local_addr":"[::]:4676","unit":"api-server","latency_us":25,"response_code":"200"}
Dec 18 22:14:44.634 INFO request completed, latency_us: 296, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: fedfb49b-fb68-4336-99e9-f54032c4f0a2, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:45.634 INFO request completed, latency_us: 412, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: d432dba6-42fe-43db-800b-932b168bcf8b, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:46.635 INFO request completed, latency_us: 458, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: db702f5d-0040-4de7-9b25-05f705c26ad3, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:47.634 INFO request completed, latency_us: 306, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 86138e3a-d463-48a0-9a52-59cc4eea72aa, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:48.634 INFO request completed, latency_us: 305, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 0d4bcc49-cfd4-40ae-a6f7-1aa853115bcb, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:49.634 INFO request completed, latency_us: 350, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 66efbbfc-b8d9-4eda-8925-c54b15be655d, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:50.634 INFO request completed, latency_us: 300, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: a4b67531-8568-4048-8d88-ded178b63e7a, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:51.634 INFO request completed, latency_us: 349, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: f4fed329-2bb8-43b0-a591-f7bdf0d38e48, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:52.634 INFO request completed, latency_us: 313, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 6982bd8d-c05c-484e-a35f-497848135661, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:53.635 INFO request completed, latency_us: 305, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: d702c1ae-3f3f-48a6-8e9c-04803817813e, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:54.634 INFO request completed, latency_us: 331, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 97da1769-70ac-4569-ac03-fa4744b0d891, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:55.634 INFO request completed, latency_us: 300, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: fad2e98d-fe07-4c95-b836-b98a58852797, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:56.634 INFO request completed, latency_us: 315, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 773688fa-5fe5-4313-8fae-87828c444df7, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:57.634 INFO request completed, latency_us: 312, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 807bbb6b-ba65-4236-9c05-492a2afc9f8c, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:58.635 INFO request completed, latency_us: 547, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: ef2eb88d-233e-4831-895d-db9bbdeb27fc, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:14:59.635 INFO request completed, latency_us: 507, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 1fa4dc00-d184-4d29-b355-f6d996e665b8, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:15:00.634 INFO request completed, latency_us: 439, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 3999b477-476e-40cc-a688-bba49b381c7b, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950
Dec 18 22:15:01.634 INFO request completed, latency_us: 330, response_code: 200, uri: /e38ae931-fe39-47a5-a036-196300c6bb27, method: GET, req_id: 4ab953d7-0de6-4156-af06-53675b17debc, remote_addr: [fd00:1122:3344:101::d]:57437, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950