[ Jun 22 01:55:00 Disabled. ]
[ Jun 22 01:55:00 Rereading configuration. ]
[ Jun 22 01:55:12 Rereading configuration. ]
[ Jun 22 01:55:13 Rereading configuration. ]
[ Jun 22 01:55:13 Enabled. ]
[ Jun 22 01:55:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Jun 22 01:55:13 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.078630618Z","hostname":"oxz_switch","pid":4179}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.095838819Z","hostname":"oxz_switch","pid":4179}
{"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.09587764Z","hostname":"oxz_switch","pid":4179}
{"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.0959373Z","hostname":"oxz_switch","pid":4179}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099809629Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.09987303Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.09988291Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.09989391Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.09990291Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099913741Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099922381Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099931651Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099945551Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099955451Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099967821Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099978422Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.099991392Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100000132Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100013882Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100023622Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100032912Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100045812Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100055673Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100066543Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100075243Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100089153Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100099103Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100110093Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100118984Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100129234Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100143204Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100152584Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100165474Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100175014Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100188385Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100197815Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.100210275Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"PUT"}
{"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.100220675Z","hostname":"oxz_switch","pid":4179}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:13.100228435Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:13.101469914Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"get originated tunnel endpoints: Communication Error: error sending request for url (http://localhost:8000/originated_tunnel_endpoints): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"slog-rs","level":50,"time":"2024-06-22T01:55:13.168205444Z","hostname":"oxz_switch","pid":4179}
[ Jun 22 01:55:14 Rereading configuration. ]
[ Jun 22 01:55:14 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 433"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=433
+ getopts c: o
+ [[ -n 433 ]]
+ pgrep -c 433
+ PARENT_PID=4178
+ [[ -n 4178 ]]
+ pgrep -P 4178
+ CHILD_PID=4179
+ [[ -n 4179 ]]
+ kill -USR1 4179
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.54190568Z","hostname":"oxz_switch","pid":4179}
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.543338971Z","hostname":"oxz_switch","pid":4179}
[ Jun 22 01:55:14 Method "refresh" exited with status 0. ]
{"msg":"resolving nexus","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.54388539Z","hostname":"oxz_switch","pid":4179}
{"msg":"new DNS resolver","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.544029252Z","hostname":"oxz_switch","pid":4179,"addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.735104864Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:36318"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.735785005Z","hostname":"oxz_switch","pid":4179,"uri":"/static/route4","method":"PUT","req_id":"e566c4cb-e564-4623-b1ee-dd05bb02eecc","remote_addr":"[fd00:1122:3344:101::1]:36318","local_addr":"[::]:4676","unit":"api-server","latency_us":445,"response_code":"204"}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2024-06-22T01:55:14.736370334Z","hostname":"oxz_switch","pid":4179}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:55:14.736380694Z","hostname":"oxz_switch","pid":4179}
{"msg":"lookup_socket_v6 srv","v":0,"name":"slog-rs","level":20,"time":"2024-06-22T01:55:19.547101648Z","hostname":"oxz_switch","pid":4179,"response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"59cebf87-1ddd-4cf3-8dfe-af47c91c765f.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"88d03fa6-91e4-44cd-bd2f-3ca495b76e2c.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"b7f02198-e503-4542-8190-5c4bb0e37c4d.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"b7f02198-e503-4542-8190-5c4bb0e37c4d.host.control-plane.oxide.internal.\"), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 926, tv_nsec: 370017838 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
Jun 22 01:55:19.549 DEBG registered DTrace probes
Jun 22 01:55:19.549 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jun 22 01:55:19.549 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:196
Jun 22 01:55:19.549 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jun 22 01:55:19.549 DEBG Nexus IP provided explicitly, will use it, addr: [fd00:1122:3344:101::c]:12221
Jun 22 01:55:19.549 DEBG starting producer registration task
Jun 22 01:55:19.549 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 1527a180-19bf-4e4e-8800-1c2109abbb81, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:276
Jun 22 01:55:19.550 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task
Jun 22 01:55:19.550 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jun 22 01:55:19.581 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:55:35.061 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:55:35.062 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 282.028798ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:55:35.442 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:55:50.442 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:55:50.444 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 405.175268ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:55:50.893 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:56:05.895 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:56:05.896 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 889.727613ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:56:06.816 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:56:21.817 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:56:21.818 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 2.912182684s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:56:24.761 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:56:39.322 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:56:39.323 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 3.622165213s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:56:42.976 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:56:57.982 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:56:57.983 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 9.553897886s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:57:07.594 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:57:22.635 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:57:22.637 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 16.309339392s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:57:39.021 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:57:54.062 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:57:54.063 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 44.038418742s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 22 01:58:38.687 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 22 01:58:53.696 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::c)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 22 01:58:53.706 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out", delay: 91.481326725s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:04.473154286Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:62078"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:04.477612654Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"83a74661-7dff-4952-b0ac-e4fecc75f400","remote_addr":"[fd00:1122:3344:101::a]:62078","local_addr":"[::]:4676","unit":"api-server","latency_us":3661,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:05.356800472Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:55799"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:05.356835083Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"10453e09-81a2-482d-bc3c-b1e3436af27d","remote_addr":"[fd00:1122:3344:101::a]:55799","local_addr":"[::]:4676","unit":"api-server","latency_us":24,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:05.941555094Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:44327"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:05.941600835Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"23a8941c-d36b-47fb-86d3-9b4f65a0568c","remote_addr":"[fd00:1122:3344:101::b]:44327","local_addr":"[::]:4676","unit":"api-server","latency_us":25,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.099454865Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:34798"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.099510375Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"4b319d4c-9987-4851-9b65-617ec1457897","remote_addr":"[fd00:1122:3344:101::c]:34798","local_addr":"[::]:4676","unit":"api-server","latency_us":29,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.62891933Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:35788"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.628984861Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"d2682b8c-19dd-4dc2-b698-1d6d3d4b75cd","remote_addr":"[fd00:1122:3344:101::b]:35788","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.663556325Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:41239"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:06.663598895Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"4dd5e08f-6629-4d69-b0a6-ebed260c559b","remote_addr":"[fd00:1122:3344:101::c]:41239","local_addr":"[::]:4676","unit":"api-server","latency_us":23,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:34.552215867Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:34286"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:34.552265117Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"a53e0958-6b32-4d32-b858-b38a07ddaeb1","remote_addr":"[fd00:1122:3344:101::a]:34286","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:34.753537914Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:63542"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:34.75390051Z","hostname":"oxz_switch","pid":4179,"uri":"/static/route4","method":"GET","req_id":"5974c4ca-bdd9-4be6-9ecd-07af12052e1e","remote_addr":"[fd00:1122:3344:101::a]:63542","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":94,"response_code":"500"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.213567696Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:60970"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.213605797Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"42330c95-23af-4bc2-818a-8c7dfad0f3b9","remote_addr":"[fd00:1122:3344:101::c]:60970","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.231285994Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:56724"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.231374746Z","hostname":"oxz_switch","pid":4179,"uri":"/bfd/peers","method":"GET","req_id":"bc627e9a-b694-4c27-a623-a3471bbf56b8","remote_addr":"[fd00:1122:3344:101::b]:56724","local_addr":"[::]:4676","unit":"api-server","latency_us":14,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.25938738Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:62682"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.25943206Z","hostname":"oxz_switch","pid":4179,"uri":"/static/route4","method":"GET","req_id":"65234712-e9da-440a-a375-d183edface16","remote_addr":"[fd00:1122:3344:101::c]:62682","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":18,"response_code":"500"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.312517443Z","hostname":"oxz_switch","pid":4179,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:53104"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-22T01:59:36.312554734Z","hostname":"oxz_switch","pid":4179,"uri":"/static/route4","method":"GET","req_id":"e9f99fae-854c-4fc0-b0f7-b0f428dc9175","remote_addr":"[fd00:1122:3344:101::b]:53104","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":18,"response_code":"500"}