[ Sep 9 22:59:30 Disabled. ] [ Sep 9 22:59:30 Rereading configuration. ] [ Sep 9 22:59:30 Enabled. ] [ Sep 9 22:59:30 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Sep 9 22:59:30 Method "start" exited with status 0. ] [ Sep 9 22:59:30 Rereading configuration. ] [ Sep 9 22:59:30 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 213"). ] + CONTRACT='' + getopts c: o + CONTRACT=213 + getopts c: o + [[ -n 213 ]] + pgrep -c 213 + PARENT_PID=2274 + [[ -n 2274 ]] + pgrep -P 2274 + CHILD_PID=2275 + [[ -n 2275 ]] + kill -USR1 2275 [ Sep 9 22:59:30 Method "refresh" exited with status 0. ] [ Sep 9 22:59:30 Stopping because all processes in service exited. ] [ Sep 9 22:59:30 Executing stop method (:kill). ] [ Sep 9 22:59:30 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ] [ Sep 9 22:59:30 Method "start" exited with status 0. ] {"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.032020033Z","hostname":"oxz_switch","pid":2302} {"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.050575228Z","hostname":"oxz_switch","pid":2302} {"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.0506843Z","hostname":"oxz_switch","pid":2302} {"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.050725851Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-09-09T22:59:31.051346122Z","hostname":"oxz_switch","pid":2302} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052632185Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052687826Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052699506Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052707246Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052717056Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers/{addr}","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052726096Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/clear/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052736146Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052744116Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052757317Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052765497Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052776897Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052785097Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052795027Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052803117Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052813888Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052821908Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052836088Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052843338Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052852938Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052862189Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052869469Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052889029Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052896719Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052906659Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/routers","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052913929Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05292681Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05293483Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05294479Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"PUT"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05295217Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/message-history","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05296154Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/omicron/apply","method":"POST"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.05296902Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/exported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052977371Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/imported","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052985031Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/neighbors","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.052993751Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/selected","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.053009651Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"DELETE"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.053019401Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"GET"} {"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.053026631Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"PUT"} {"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.053035741Z","hostname":"oxz_switch","pid":2302} {"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:31.053041312Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2025-09-09T22:59:31.053047302Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server"} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:31.05353646Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:32.054049086Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:33.054487331Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:34.054803004Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:35.055153187Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:36.055563521Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:37.056082947Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:38.056366639Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:39.056798004Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:40.057130336Z","hostname":"oxz_switch","pid":2302} {"msg":"failed to ensure TEP address fd3f:4470:eb49:392e::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-09-09T22:59:41.057461339Z","hostname":"oxz_switch","pid":2302} [ Sep 9 22:59:42 Rereading configuration. ] [ Sep 9 22:59:42 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 218"). ] + CONTRACT='' + getopts c: o + CONTRACT=218 + getopts c: o + [[ -n 218 ]] + pgrep -c 218 + PARENT_PID=2301 + [[ -n 2301 ]] + pgrep -P 2301 + CHILD_PID=2302 + [[ -n 2302 ]] + kill -USR1 2302 {"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:42.260014941Z","hostname":"oxz_switch","pid":2302} {"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:42.261436715Z","hostname":"oxz_switch","pid":2302} [ Sep 9 22:59:42 Method "refresh" exited with status 0. ] Sep 09 22:59:42.262 DEBG registered DTrace probes Sep 09 22:59:42.262 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Sep 09 22:59:42.262 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.3/src/server.rs:328 Sep 09 22:59:42.262 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot Sep 09 22:59:42.262 DEBG Nexus IP not provided, will use DNS to resolve it Sep 09 22:59:42.262 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/9f048ca/internal-dns/resolver/src/resolver.rs:144 Sep 09 22:59:42.263 DEBG starting producer registration task Sep 09 22:59:42.263 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 779be585-b346-4e26-942a-354ce9098a3f, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/9f048ca/oximeter/producer/src/lib.rs:279 Sep 09 22:59:42.263 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task {"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:42.567461865Z","hostname":"oxz_switch","pid":2302,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:58604"} {"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:42.568008595Z","hostname":"oxz_switch","pid":2302,"uri":"/static/route4","method":"PUT","req_id":"9de35dc3-ff0b-4758-b847-e955d35ddd00","remote_addr":"[fd00:1122:3344:101::1]:58604","local_addr":"[::]:4676","unit":"api-server","latency_us":357,"response_code":204} {"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2025-09-09T22:59:42.569019593Z","hostname":"oxz_switch","pid":2302} {"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2025-09-09T22:59:42.569037363Z","hostname":"oxz_switch","pid":2302} Sep 09 22:59:47.265 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4245cf9b-2641-419d-b4cc-35dbc752b37c.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("db3336a3-749d-428c-8e3d-47bce2e3fb2e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("dd41ddb7-0042-4648-8197-b26b507b992b.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("dd41ddb7-0042-4648-8197-b26b507b992b.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 7489, tv_nsec: 860349212 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver Sep 09 22:59:47.266 DEBG will register with Nexus at [fd00:1122:3344:101::b]:12221, component: producer-registration-task Sep 09 22:59:47.266 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::b]:12221/metrics/producers, method: POST, component: producer-registration-task