[ Jun 20 21:23:33 Disabled. ]
[ Jun 20 21:23:33 Rereading configuration. ]
[ Jun 20 21:23:46 Rereading configuration. ]
[ Jun 20 21:23:46 Rereading configuration. ]
[ Jun 20 21:23:46 Enabled. ]
[ Jun 20 21:23:46 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Jun 20 21:23:46 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.572089618Z","hostname":"oxz_switch","pid":4193}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.591303346Z","hostname":"oxz_switch","pid":4193}
{"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.591383678Z","hostname":"oxz_switch","pid":4193}
{"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.591434069Z","hostname":"oxz_switch","pid":4193}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595222425Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595280887Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595291327Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595303597Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595313608Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595325218Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595334478Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595345238Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595357899Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595368379Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595382489Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595391919Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.5954113Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.5954212Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.59543436Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595443891Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595453641Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595466231Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595475422Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595486232Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595496752Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595509762Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595519523Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595532003Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595541413Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595551783Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595563704Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595574324Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595586184Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595596865Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595640386Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595657766Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.595674557Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","path":"/static/route4","method":"PUT"}
{"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.595686607Z","hostname":"oxz_switch","pid":4193}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:46.595695067Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:46.596817076Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"get originated tunnel endpoints: Communication Error: error sending request for url (http://localhost:8000/originated_tunnel_endpoints): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"slog-rs","level":50,"time":"2024-06-20T21:23:46.66429226Z","hostname":"oxz_switch","pid":4193}
[ Jun 20 21:23:48 Rereading configuration. ]
[ Jun 20 21:23:48 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 434"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=434
+ getopts c: o
+ [[ -n 434 ]]
+ pgrep -c 434
+ PARENT_PID=4192
+ [[ -n 4192 ]]
+ pgrep -P 4192
+ CHILD_PID=4193
+ [[ -n 4193 ]]
+ kill -USR1 4193
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.034823638Z","hostname":"oxz_switch","pid":4193}
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.036235514Z","hostname":"oxz_switch","pid":4193}
[ Jun 20 21:23:48 Method "refresh" exited with status 0. ]
{"msg":"resolving nexus","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.036718956Z","hostname":"oxz_switch","pid":4193}
{"msg":"new DNS resolver","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.03687237Z","hostname":"oxz_switch","pid":4193,"addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.226251261Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:65168"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.226782625Z","hostname":"oxz_switch","pid":4193,"uri":"/static/route4","method":"PUT","req_id":"a2250fd4-7946-45a4-af1e-9c45af037cf7","remote_addr":"[fd00:1122:3344:101::1]:65168","local_addr":"[::]:4676","unit":"api-server","latency_us":373,"response_code":"204"}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2024-06-20T21:23:48.228026966Z","hostname":"oxz_switch","pid":4193}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:23:48.228045227Z","hostname":"oxz_switch","pid":4193}
{"msg":"resolving nexus failed, no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN } waiting 146.891381ms","v":0,"name":"slog-rs","level":40,"time":"2024-06-20T21:23:53.039573543Z","hostname":"oxz_switch","pid":4193}
{"msg":"lookup_socket_v6 srv","v":0,"name":"slog-rs","level":20,"time":"2024-06-20T21:23:53.187700596Z","hostname":"oxz_switch","pid":4193,"response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"65b469d2-6eb7-4e46-8276-ad6d27b67bf9.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"8f64d0bd-6f7c-418c-a532-2ff9143ec947.host.control-plane.oxide.internal.\") })) }, Record { name_labels: Name(\"8f64d0bd-6f7c-418c-a532-2ff9143ec947.host.control-plane.oxide.internal.\"), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 639, tv_nsec: 750573891 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"}
Jun 20 21:23:53.189 DEBG registered DTrace probes
Jun 20 21:23:53.189 DEBG registered endpoint, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jun 20 21:23:53.189 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:196
Jun 20 21:23:53.189 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jun 20 21:23:53.189 DEBG Nexus IP provided explicitly, will use it, addr: [fd00:1122:3344:101::a]:12221
Jun 20 21:23:53.189 DEBG starting producer registration task
Jun 20 21:23:53.189 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: e789eeb5-b9d0-4486-9315-eafe6d7ad753, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:276
Jun 20 21:23:53.189 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task
Jun 20 21:23:53.189 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Jun 20 21:23:53.221 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:07.827 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:07.828 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 321.718673ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:08.180 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:23.180 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:23.181 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 738.250506ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:23.951 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:39.668 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:39.669 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 1.387965318s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:41.086 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:24:56.046 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:24:56.047 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 1.426872618s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:24:57.500 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:25:12.461 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:12.462 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 2.045376969s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:25:14.532 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:25:29.492 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:29.494 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 8.706693007s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:25:38.207 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:25:53.168 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:25:53.170 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 13.529884995s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:26:06.694 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:26:21.654 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:26:21.656 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 19.41991158s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:26:41.053 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:26:56.013 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:26:56.015 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 81.941320872s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
Jun 20 21:28:18.254 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:28:33.258 DEBG client response, result: Err(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, source: TimedOut }), component: producer-registration-task
Jun 20 21:28:33.264 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out", delay: 86.397718544s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/450f906/oximeter/producer/src/lib.rs:417
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:51.173371424Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:54756"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:51.1791209Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"1ff1dbbc-7ed5-422b-afed-5f826ed615cb","remote_addr":"[fd00:1122:3344:101::b]:54756","local_addr":"[::]:4676","unit":"api-server","latency_us":4841,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:51.909328241Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:57024"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:51.909379072Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"b4946c07-a539-4102-a99c-9d29437f3702","remote_addr":"[fd00:1122:3344:101::b]:57024","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.027529843Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:38084"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.027587495Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"73d711d0-d5e5-4283-9bee-ed7c458ffab7","remote_addr":"[fd00:1122:3344:101::c]:38084","local_addr":"[::]:4676","unit":"api-server","latency_us":33,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.177190145Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:34786"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.177241737Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"e79dcac1-f6d2-4028-bcde-da6893c23d56","remote_addr":"[fd00:1122:3344:101::a]:34786","local_addr":"[::]:4676","unit":"api-server","latency_us":30,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.568830045Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:36062"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.568879876Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"f5bc2471-5deb-456e-945f-40b61acb9c4a","remote_addr":"[fd00:1122:3344:101::c]:36062","local_addr":"[::]:4676","unit":"api-server","latency_us":29,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.617519422Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:40999"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:28:52.617596124Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"3f5b5836-4bfe-4fb2-843b-87f74443cc24","remote_addr":"[fd00:1122:3344:101::a]:40999","local_addr":"[::]:4676","unit":"api-server","latency_us":30,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.152209893Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:55818"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.152257384Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"b2781e6e-5198-4d60-bb0f-d64d8dbc5219","remote_addr":"[fd00:1122:3344:101::b]:55818","local_addr":"[::]:4676","unit":"api-server","latency_us":27,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.938919068Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:64269"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.93896219Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"2c55839d-3f98-43fa-8778-27203337e7e0","remote_addr":"[fd00:1122:3344:101::c]:64269","local_addr":"[::]:4676","unit":"api-server","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.955360326Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:40761"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:21.955429158Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"73d4a1d5-55b6-4e6a-9089-1344c7ec5991","remote_addr":"[fd00:1122:3344:101::a]:40761","local_addr":"[::]:4676","unit":"api-server","latency_us":15,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:51.662202623Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:33941"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:51.662270685Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"cb5c91bb-da2b-4fc8-ba49-432a8f6937f0","remote_addr":"[fd00:1122:3344:101::b]:33941","local_addr":"[::]:4676","unit":"api-server","latency_us":25,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:51.815328103Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:57826"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:51.816881482Z","hostname":"oxz_switch","pid":4193,"uri":"/static/route4","method":"GET","req_id":"78b78bc5-eda8-4f4c-a97b-ac77e471fba7","remote_addr":"[fd00:1122:3344:101::b]:57826","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":1054,"response_code":"500"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.283923448Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:61365"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.28400776Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"ce76559a-634a-4353-95ae-78fa8a06237b","remote_addr":"[fd00:1122:3344:101::c]:61365","local_addr":"[::]:4676","unit":"api-server","latency_us":37,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.287840077Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:35123"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.287904769Z","hostname":"oxz_switch","pid":4193,"uri":"/bfd/peers","method":"GET","req_id":"8c2c1a4f-3690-4601-9fb1-2a66acf34e8f","remote_addr":"[fd00:1122:3344:101::a]:35123","local_addr":"[::]:4676","unit":"api-server","latency_us":13,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.499281269Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:59376"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.49933062Z","hostname":"oxz_switch","pid":4193,"uri":"/static/route4","method":"GET","req_id":"a5814c3e-e638-4a35-bec4-6f2acfc3306d","remote_addr":"[fd00:1122:3344:101::a]:59376","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":31,"response_code":"500"}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.579558118Z","hostname":"oxz_switch","pid":4193,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:53870"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2024-06-20T21:29:52.579608299Z","hostname":"oxz_switch","pid":4193,"uri":"/static/route4","method":"GET","req_id":"50c94435-1e6b-437c-9ae0-f06ac526ec7f","remote_addr":"[fd00:1122:3344:101::c]:53870","local_addr":"[::]:4676","unit":"api-server","error_message_external":"Internal Server Error","error_message_internal":"key must be a string","latency_us":26,"response_code":"500"}
Jun 20 21:29:59.699 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Jun 20 21:29:59.760 DEBG client response, result: Ok(Response { url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv6(fd00:1122:3344:101::a)), port: Some(12221), path: "/metrics/producers", query: None, fragment: None }, status: 201, headers: {"content-type": "application/json", "x-request-id": "744ae1c9-7f0d-4e6e-89bf-a29f3e9eebb4", "content-length": "41", "date": "Thu, 20 Jun 2024 21:29:59 GMT"} }), component: producer-registration-task
Jun 20 21:29:59.762 DEBG registered with nexus successfully, component: producer-registration-task
Jun 20 21:29:59.762 DEBG pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
Jun 20 21:30:00.791 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:45914, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:766
Jun 20 21:30:00.807 INFO request completed, latency_us: 15249, response_code: 200, uri: /e789eeb5-b9d0-4486-9315-eafe6d7ad753, method: GET, req_id: 0579fbf2-dfcd-47ff-a31f-8a99082ef8a3, remote_addr: [fd00:1122:3344:101::d]:45914, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0560bed/dropshot/src/server.rs:878