[ Apr 15 03:58:26 Disabled. ] [ Apr 15 03:58:26 Rereading configuration. ] [ Apr 15 03:58:27 Enabled. ] [ Apr 15 03:58:27 Rereading configuration. ] [ Apr 15 03:58:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Apr 15 03:58:27 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.702853856Z","hostname":"oxz_switch","pid":2275} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.72506593Z","hostname":"oxz_switch","pid":2275} {"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.725177813Z","hostname":"oxz_switch","pid":2275} {"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.725297626Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-04-15T03:58:27.725757185Z","hostname":"oxz_switch","pid":2275} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727046713Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727089154Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727104524Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers/{addr}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727114434Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/clear/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727122984Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727134555Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727143555Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727154875Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727163015Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727187826Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727196656Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727209896Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727218916Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727230797Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727237957Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727249937Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727258267Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727274918Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727284538Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727295148Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727302848Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727321859Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/routers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727331669Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727342579Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727351409Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.72736876Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.7273785Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/message-history","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.72739277Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/omicron/apply","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.7274023Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/exported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727453401Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/imported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727484642Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727495822Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/selected","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727508913Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727519273Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.727531813Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"PUT"} {"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.727541173Z","hostname":"oxz_switch","pid":2275} {"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:27.727551823Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:27.728409292Z","hostname":"oxz_switch","pid":2275} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2025-04-15T03:58:27.728575795Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"get originated tunnel endpoints: Communication Error: error sending request for url (http://localhost:8000/originated_tunnel_endpoints)","v":0,"name":"slog-rs","level":50,"time":"2025-04-15T03:58:27.728609806Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:28.729029993Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:29.731015923Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:30.731726356Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:31.732249144Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:32.732807783Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:33.733345002Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:34.733885561Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:35.734427151Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:36.7349419Z","hostname":"oxz_switch","pid":2275} {"msg":"failed to ensure TEP address fd27:aabd:8df:5947::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-04-15T03:58:37.73549259Z","hostname":"oxz_switch","pid":2275} [ Apr 15 03:58:38 Rereading configuration. ] [ Apr 15 03:58:38 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2274 + [[ -n 2274 ]] + pgrep -P 2274 + CHILD_PID=2275 + [[ -n 2275 ]] + kill -USR1 2275 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:38.983528519Z","hostname":"oxz_switch","pid":2275} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:38.984850868Z","hostname":"oxz_switch","pid":2275} [ Apr 15 03:58:38 Method "refresh" exited with status 0. ] Apr 15 03:58:38.985 DEBG registered DTrace probes Apr 15 03:58:38.985 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Apr 15 03:58:38.986 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.16.0/src/server.rs:328 Apr 15 03:58:38.986 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Apr 15 03:58:38.986 DEBG Nexus IP not provided, will use DNS to resolve it Apr 15 03:58:38.986 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/internal-dns/resolver/src/resolver.rs:111 Apr 15 03:58:38.986 DEBG starting producer registration task Apr 15 03:58:38.986 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 3950708e-696d-4ac2-9e28-69075bae5884, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/oximeter/producer/src/lib.rs:279 Apr 15 03:58:38.986 DEBG registering / renewing oximeter producer lease with Nexus, component: producer-registration-task {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:39.40019366Z","hostname":"oxz_switch","pid":2275,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:57071"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:39.400811453Z","hostname":"oxz_switch","pid":2275,"uri":"/static/route4","method":"PUT","req_id":"1ce8142e-884b-455e-88af-89aab26fc2dc","remote_addr":"[fd00:1122:3344:101::1]:57071","local_addr":"[::]:4676","unit":"api-server","latency_us":418,"response_code":"204"} {"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2025-04-15T03:58:39.401292554Z","hostname":"oxz_switch","pid":2275} {"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2025-04-15T03:58:39.401304014Z","hostname":"oxz_switch","pid":2275} Apr 15 03:58:48.990 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("527069c3-e732-4ddb-a5bc-380dcdd7efde.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("909a489c-1cdb-4440-bdff-c8667de265bf.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), rr_type: SRV, dns_class: IN, ttl: 0, rdata: Some(SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("c2373202-6a37-4079-9fde-976f12481944.host.control-plane.oxide.internal.") })) }, Record { name_labels: Name("c2373202-6a37-4079-9fde-976f12481944.host.control-plane.oxide.internal."), rr_type: AAAA, dns_class: IN, ttl: 0, rdata: Some(AAAA(AAAA(fd00:1122:3344:101::a))) }], valid_until: Instant { tv_sec: 1630, tv_nsec: 931915887 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Apr 15 03:58:48.990 DEBG using nexus address for registration, addr: [fd00:1122:3344:101::c]:12221, component: producer-registration-task Apr 15 03:58:48.990 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 15 03:59:03.803 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 15 03:59:03.803 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 202.868865ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/oximeter/producer/src/lib.rs:420 Apr 15 03:59:04.007 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 15 03:59:19.049 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 15 03:59:19.049 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 333.344091ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/oximeter/producer/src/lib.rs:420 Apr 15 03:59:19.385 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 15 03:59:34.387 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 15 03:59:34.387 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 640.872623ms, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/oximeter/producer/src/lib.rs:420 Apr 15 03:59:35.029 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task Apr 15 03:59:50.030 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task Apr 15 03:59:50.030 WARN failed to register as a producer with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 1.314126395s, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/77c4136/oximeter/producer/src/lib.rs:420 Apr 15 03:59:51.346 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task