[ Dec 11 18:12:25 Disabled. ]
[ Dec 11 18:12:25 Rereading configuration. ]
[ Dec 11 18:12:25 Enabled. ]
[ Dec 11 18:12:25 Rereading configuration. ]
[ Dec 11 18:12:25 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Dec 11 18:12:26 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.051249272Z","hostname":"oxz_switch","pid":2287}
{"msg":"signal handler waiting got context","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.072534241Z","hostname":"oxz_switch","pid":2287}
{"msg":"bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.072602653Z","hostname":"oxz_switch","pid":2287}
{"msg":"bfd peers: []","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.072626943Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2025-12-11T18:12:26.073412681Z","hostname":"oxz_switch","pid":2287}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.075914107Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.075944248Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bestpath/config/fanout","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.075965118Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.075975878Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076009739Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076022239Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/clear/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07603099Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07605066Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07606214Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07607592Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076084441Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076098731Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076111111Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076125442Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076136012Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076150862Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076167853Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076188613Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076201543Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076223304Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076235254Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076264915Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076277005Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076293656Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076306166Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076326986Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076340266Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076361287Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076373667Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076390018Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076401798Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/exported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076413628Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076441729Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076453249Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.076463949Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07647122Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07648415Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","versions":"all","path":"/static/route4","method":"PUT"}
{"msg":"admin: listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.07649143Z","hostname":"oxz_switch","pid":2287}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:26.07650051Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2025-12-11T18:12:26.07650733Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server"}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:26.076516921Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:27.076789819Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:28.077140439Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:29.077363406Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:30.077736546Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:31.078044785Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:32.078471057Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:33.078839257Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:34.079172696Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:35.07967241Z","hostname":"oxz_switch","pid":2287}
{"msg":"failed to ensure TEP address fd8c:78dd:4294:d6bc::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2025-12-11T18:12:36.079953129Z","hostname":"oxz_switch","pid":2287}
[ Dec 11 18:12:37 Rereading configuration. ]
[ Dec 11 18:12:37 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 214"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=214
+ getopts c: o
+ [[ -n 214 ]]
+ pgrep -c 214
+ PARENT_PID=2285
+ [[ -n 2285 ]]
+ pgrep -P 2285
+ CHILD_PID=2287
+ [[ -n 2287 ]]
+ kill -USR1 2287
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:37.405411374Z","hostname":"oxz_switch","pid":2287}
[ Dec 11 18:12:37 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:37.40701791Z","hostname":"oxz_switch","pid":2287}
Dec 11 18:12:37.407 DEBG registered DTrace probes
Dec 11 18:12:37.408 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Dec 11 18:12:37.408 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328
Dec 11 18:12:37.408 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Dec 11 18:12:37.408 DEBG Nexus IP not provided, will use DNS to resolve it
Dec 11 18:12:37.408 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/internal-dns/resolver/src/resolver.rs:144
Dec 11 18:12:37.408 DEBG starting producer registration task
Dec 11 18:12:37.408 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: 866956d3-f24f-4fc2-9796-731342d8440d, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:279
Dec 11 18:12:37.408 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:37.758419438Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::1]:57349"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:37.759272557Z","hostname":"oxz_switch","pid":2287,"uri":"/static/route4","method":"PUT","req_id":"d847bdca-4ecd-4d27-9ace-1ad20b6f64ca","remote_addr":"[fd00:1122:3344:101::1]:57349","local_addr":"[::]:4676","unit":"api-server","latency_us":582,"response_code":204}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2025-12-11T18:12:37.760492975Z","hostname":"oxz_switch","pid":2287}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:12:37.760514255Z","hostname":"oxz_switch","pid":2287}
Dec 11 18:12:42.411 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1675, tv_nsec: 494953653 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:12:42.411 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:12:42.412 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:12:57.415 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:12:57.415 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 20.005542799s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:12:57.564 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1690, tv_nsec: 647243253 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:12:57.565 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:12:57.565 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:13:12.607 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:13:12.607 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 35.155637791s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:13:13.014 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1706, tv_nsec: 54108952 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:13:13.014 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:13:13.015 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:13:28.042 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:13:28.046 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 50.573406376s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:13:29.304 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1722, tv_nsec: 323931742 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:13:29.306 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:13:29.319 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:13:44.320 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:13:44.320 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 66.848093584s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:13:46.591 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1739, tv_nsec: 611770257 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:13:46.592 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:13:46.592 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:14:01.599 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:14:01.602 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 84.126106128s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:14:05.080 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1758, tv_nsec: 85235362 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:14:05.221 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:14:05.224 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:14:20.225 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Dec 11 18:14:20.225 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 102.752295038s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
Dec 11 18:14:25.895 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1778, tv_nsec: 915305695 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:14:25.896 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:14:25.896 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:14:33.801 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", [fd00:1122:3344:101::a]:12221, Os { code: 146, kind: ConnectionRefused, message: "Connection refused" })) }), component: producer-registration-task
Dec 11 18:14:33.801 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)", delay: 116.328281309s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/5b8b822/oximeter/producer/src/lib.rs:371
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:42.661671944Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::b]:38521"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:42.672551409Z","hostname":"oxz_switch","pid":2287,"uri":"/bfd/peers","method":"GET","req_id":"66cf6481-bd0b-4015-9a74-89974bb3375c","remote_addr":"[fd00:1122:3344:101::b]:38521","local_addr":"[::]:4676","unit":"api-server","latency_us":9082,"response_code":200}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:43.914113586Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::a]:40787"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:43.914313221Z","hostname":"oxz_switch","pid":2287,"uri":"/bfd/peers","method":"GET","req_id":"708897bb-2fe1-4130-bf21-13f57aa1261e","remote_addr":"[fd00:1122:3344:101::a]:40787","local_addr":"[::]:4676","unit":"api-server","latency_us":62,"response_code":200}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:43.971235242Z","hostname":"oxz_switch","pid":2287,"local_addr":"[::]:4676","unit":"api-server","remote_addr":"[fd00:1122:3344:101::c]:60862"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2025-12-11T18:14:43.972221324Z","hostname":"oxz_switch","pid":2287,"uri":"/bfd/peers","method":"GET","req_id":"5b826f7d-3d3d-41eb-a374-a2fc888a172e","remote_addr":"[fd00:1122:3344:101::c]:60862","local_addr":"[::]:4676","unit":"api-server","latency_us":895,"response_code":200}
Dec 11 18:14:52.048 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("34187f2f-4826-4991-99b3-2a4ff2992a14.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("4ef6e9fe-b880-4221-9db8-d5a1ad5faef5.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("fd5f29f9-5ef8-4d2d-bf35-86062269b1e1.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 1805, tv_nsec: 68448738 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Dec 11 18:14:52.049 DEBG will register with Nexus at [fd00:1122:3344:101::a]:12221, component: producer-registration-task
Dec 11 18:14:52.049 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::a]:12221/metrics/producers, method: POST, component: producer-registration-task
Dec 11 18:14:52.102 DEBG client response, result: Ok(Response { url: "http://[fd00:1122:3344:101::a]:12221/metrics/producers", status: 201, headers: {"content-type": "application/json", "x-request-id": "00a58ffe-6199-4e7a-b2c9-15f12a8bd39e", "content-length": "41", "date": "Thu, 11 Dec 2025 18:14:52 GMT"} }), component: producer-registration-task
Dec 11 18:14:52.103 DEBG successfully registered with Nexus, pausing until time to renew lease, wait_period: 150s, lease_duration: 600s, component: producer-registration-task
Dec 11 18:14:54.999 INFO accepted connection, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:1025
Dec 11 18:14:55.010 INFO request completed, latency_us: 10576, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 79829dff-ab0c-422a-9518-0270a29663ce, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:14:55.632 INFO request completed, latency_us: 401, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: eaf7bd7a-9b53-459e-8042-bc724aa5e7cb, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:14:56.631 INFO request completed, latency_us: 368, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: c11a68ec-5452-4bbf-8c28-cb4fd9f86e37, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:14:57.631 INFO request completed, latency_us: 379, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 8ea96d76-f161-4127-895c-b57f1214ea10, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:14:58.631 INFO request completed, latency_us: 594, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 1c8b1044-32ff-45d1-859a-26ebf2ce90c5, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:14:59.632 INFO request completed, latency_us: 499, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 937fff9b-6594-411f-9c78-2079a9e33dd0, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:15:00.632 INFO request completed, latency_us: 374, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 80117cfe-4e45-4c52-a120-d68370e1d962, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867
Dec 11 18:15:01.632 INFO request completed, latency_us: 375, response_code: 200, uri: /866956d3-f24f-4fc2-9796-731342d8440d, method: GET, req_id: 16d16d1a-1107-47e8-8206-4f0d0cd824a6, remote_addr: [fd00:1122:3344:101::d]:63694, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867