[ Jan 30 03:41:32 Disabled. ]
[ Jan 30 03:41:32 Rereading configuration. ]
[ Jan 30 03:41:33 Enabled. ]
[ Jan 30 03:41:33 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Jan 30 03:41:33 Method "start" exited with status 0. ]
[ Jan 30 03:41:33 Rereading configuration. ]
[ Jan 30 03:41:33 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 211"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=211
+ getopts c: o
+ [[ -n 211 ]]
+ pgrep -c 211
+ PARENT_PID=2271
+ [[ -n 2271 ]]
+ pgrep -P 2271
+ CHILD_PID=2272
+ [[ -n 2272 ]]
+ kill -USR1 2272
[ Jan 30 03:41:33 Method "refresh" exited with status 0. ]
[ Jan 30 03:41:33 Stopping because all processes in service exited. ]
[ Jan 30 03:41:33 Executing stop method (:kill). ]
[ Jan 30 03:41:33 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/mgd/pkg/mgd_method_script.sh &"). ]
[ Jan 30 03:41:33 Method "start" exited with status 0. ]
{"msg":"signal handler waiting for context","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.23266607Z","hostname":"oxz_switch","pid":2299,"unit":"signal","module":"admin","component":"mgd"}
{"msg":"dispatcher started","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.232934437Z","hostname":"oxz_switch","pid":2299,"listen_address":"[::]:179","unit":"dispatcher","module":"neighbor","component":"bgp"}
{"msg":"listener bind: [::]:179","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.232947907Z","hostname":"oxz_switch","pid":2299,"listen_address":"[::]:179","unit":"dispatcher","module":"neighbor","component":"bgp"}
{"msg":"signal handler got context","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.253797882Z","hostname":"oxz_switch","pid":2299,"unit":"signal","module":"admin","component":"mgd"}
{"msg":"starting bgp routers: {}","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.253944336Z","hostname":"oxz_switch","pid":2299,"unit":"daemon","module":"admin","component":"mgd"}
{"msg":"starting bfd sessions: []","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.253979737Z","hostname":"oxz_switch","pid":2299,"unit":"daemon","module":"admin","component":"mgd"}
{"msg":"failed to start stats server: underlay address not found","v":0,"name":"slog-rs","level":50,"time":"2026-01-30T03:41:33.254586212Z","hostname":"oxz_switch","pid":2299,"error":"underlay address not found","params":"hostname oxz_switch, rack 1db1e07f-6540-41dd-8e95-2450ae881aea, sled 7c269bf7-eb13-42a1-b02f-6f18599a21a5","unit":"daemon","module":"admin","component":"mgd"}
{"msg":"client request","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.255158487Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","body":"None","uri":"http://[::1]:12225/local/switch-id","method":"GET"}
{"msg":"client response","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.255914826Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","result":"Err(reqwest::Error { kind: Request, url: \"http://[::1]:12225/local/switch-id\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [::1]:12225, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to resolve switch slot","v":0,"name":"slog-rs","level":50,"time":"2026-01-30T03:41:33.255929587Z","hostname":"oxz_switch","pid":2299,"error":"Communication Error: error sending request for url (http://[::1]:12225/local/switch-id)"}
{"msg":"listening on [::]:4676","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.25605468Z","hostname":"oxz_switch","pid":2299}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256189753Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bestpath/config/fanout","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256219004Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bestpath/config/fanout","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256230704Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bfd/peers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256239695Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bfd/peers","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256250665Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bfd/peers/{addr}","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256259085Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/clear/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256269305Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/checker","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256279576Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/checker","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256287246Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/checker","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256297636Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/checker","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256305426Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/neighbor","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256318137Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/neighbor","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256327227Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/neighbor","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256348367Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/neighbor","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256357588Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256387178Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/origin4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256396839Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/origin4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256409359Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/origin4","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256420109Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/origin4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256428839Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/config/origin6","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.25644317Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/config/origin6","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.2564529Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/config/origin6","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.25646336Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/config/origin6","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256471531Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/router","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256483991Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/router","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256496661Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/router","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256504831Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/router","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256514492Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/routers","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256522462Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/shaper","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256532472Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/shaper","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256540352Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/shaper","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256554503Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/config/shaper","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256563023Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/history/fsm","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256574443Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/history/message","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256583863Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bgp/message-history","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256591854Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/omicron/apply","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256601094Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/bgp/status/exported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256608514Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bgp/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256618764Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256626264Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/bgp/status/neighbors","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256642605Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all ending with 2.0.0","path":"/bgp/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256652085Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/rib/config/bestpath/fanout","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256659635Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/rib/config/bestpath/fanout","method":"POST"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256668626Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/rib/status/imported","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256676236Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/rib/status/selected","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256686686Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/static/route4","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256697346Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/static/route4","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256705497Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all","path":"/static/route4","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256716067Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/static/route6","method":"DELETE"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256723797Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/static/route6","method":"GET"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256733437Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 2.0.0","path":"/static/route6","method":"PUT"}
{"msg":"registered endpoint","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256740947Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","versions":"all starting from 3.0.0","path":"/switch/identifiers","method":"GET"}
{"msg":"listening","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.256750408Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:33.256757508Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"successfully registered DTrace USDT probes","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:33.256767798Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd"}
{"msg":"admin: server started","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:33.256775958Z","hostname":"oxz_switch","pid":2299}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:34.256903074Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:35.257313676Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:36.257670447Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:37.257961417Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:38.258227055Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:39.258706879Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:40.259112871Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:41.259558945Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:42.259989567Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"client request","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:43.257186568Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","body":"None","uri":"http://[::1]:12225/local/switch-id","method":"GET"}
{"msg":"client response","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:43.257417534Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","result":"Err(reqwest::Error { kind: Request, url: \"http://[::1]:12225/local/switch-id\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [::1]:12225, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to resolve switch slot","v":0,"name":"slog-rs","level":50,"time":"2026-01-30T03:41:43.257465475Z","hostname":"oxz_switch","pid":2299,"error":"Communication Error: error sending request for url (http://[::1]:12225/local/switch-id)"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:43.260221126Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:44.26074638Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:45.260971008Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:46.261659757Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:47.265051256Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:48.264028391Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:49.264658038Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:50.265041979Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:51.265401011Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:52.265804413Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"client request","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:53.258496436Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","body":"None","uri":"http://[::1]:12225/local/switch-id","method":"GET"}
{"msg":"client response","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:53.258663631Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","result":"Err(reqwest::Error { kind: Request, url: \"http://[::1]:12225/local/switch-id\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [::1]:12225, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"}
{"msg":"failed to resolve switch slot","v":0,"name":"slog-rs","level":50,"time":"2026-01-30T03:41:53.258694581Z","hostname":"oxz_switch","pid":2299,"error":"Communication Error: error sending request for url (http://[::1]:12225/local/switch-id)"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:53.266083411Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
{"msg":"failed to ensure TEP address fd3a:18cc:b951:a8af::1 on ASIC: Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","v":0,"name":"slog-rs","level":40,"time":"2026-01-30T03:41:54.266434451Z","hostname":"oxz_switch","pid":2299,"prefix":"fd3a:18cc:b951:a8af::1","error":"Communication Error: error sending request for url (http://localhost:12224/loopback/ipv6)","unit":"dpd","module":"sync","component":"mg-lower"}
[ Jan 30 03:41:54 Rereading configuration. ]
[ Jan 30 03:41:54 Executing refresh method ("/opt/oxide/mgd/pkg/refresh_ctrun_child.sh -c 216"). ]
+ CONTRACT=''
+ getopts c: o
+ CONTRACT=216
+ getopts c: o
+ [[ -n 216 ]]
+ pgrep -c 216
+ PARENT_PID=2297
+ [[ -n 2297 ]]
+ pgrep -P 2297
+ CHILD_PID=2299
+ [[ -n 2299 ]]
+ kill -USR1 2299
{"msg":"handling smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:54.561103997Z","hostname":"oxz_switch","pid":2299,"unit":"smf","module":"admin","component":"mgd"}
[ Jan 30 03:41:54 Method "refresh" exited with status 0. ]
{"msg":"starting stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:54.563394826Z","hostname":"oxz_switch","pid":2299,"unit":"smf","module":"admin","component":"mgd"}
{"msg":"started stats server on smf refresh","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:54.563704164Z","hostname":"oxz_switch","pid":2299,"unit":"smf","module":"admin","component":"mgd"}
Jan 30 03:41:54.564 DEBG registered DTrace probes
Jan 30 03:41:54.564 DEBG registered endpoint, versions: all, path: /{producer_id}, method: GET, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jan 30 03:41:54.564 INFO listening, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot, file: /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.6/src/server.rs:328
Jan 30 03:41:54.565 DEBG successfully registered DTrace USDT probes, local_addr: [fd00:1122:3344:101::2]:4677, component: dropshot
Jan 30 03:41:54.565 DEBG Nexus IP not provided, will use DNS to resolve it
Jan 30 03:41:54.565 INFO new DNS resolver, addresses: [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53], component: internal-dns-resolver, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/internal-dns/resolver/src/resolver.rs:144
Jan 30 03:41:54.565 DEBG starting producer registration task
Jan 30 03:41:54.565 INFO starting oximeter metric producer server, interval: 1s, address: [fd00:1122:3344:101::2]:4677, producer_id: ae429083-1316-4b7b-8900-d01e6c3cf807, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:279
Jan 30 03:41:54.565 DEBG registering as oximeter producer with Nexus, registration_count: 0, component: producer-registration-task
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:55.074686306Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","remote_addr":"[fd00:1122:3344:101::1]:57750"}
{"msg":"determined request API version","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:41:55.07522849Z","hostname":"oxz_switch","pid":2299,"uri":"/static/route4","method":"PUT","req_id":"1e2c7bc2-e87f-415f-9aea-4e9813df53d4","remote_addr":"[fd00:1122:3344:101::1]:57750","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","version":"3.0.0"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:55.075741923Z","hostname":"oxz_switch","pid":2299,"uri":"/static/route4","method":"PUT","req_id":"1e2c7bc2-e87f-415f-9aea-4e9813df53d4","remote_addr":"[fd00:1122:3344:101::1]:57750","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","latency_us":682,"response_code":204}
{"msg":"handling change failed: libnet error route io error No such process (os error 3)","v":0,"name":"slog-rs","level":50,"time":"2026-01-30T03:41:55.07796705Z","hostname":"oxz_switch","pid":2299,"error":"libnet error route io error No such process (os error 3)","unit":"event_loop","module":"sync","component":"mg-lower"}
{"msg":"restarting sync loop","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:41:55.077985371Z","hostname":"oxz_switch","pid":2299,"unit":"event_loop","module":"sync","component":"mg-lower"}
Jan 30 03:41:59.567 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 669, tv_nsec: 113656450 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:41:59.568 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:41:59.568 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
{"msg":"client request","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:42:03.259635801Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","body":"None","uri":"http://[::1]:12225/local/switch-id","method":"GET"}
{"msg":"client response","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:42:03.26001865Z","hostname":"oxz_switch","pid":2299,"unit":"gateway-client","result":"Ok(Response { url: \"http://[::1]:12225/local/switch-id\", status: 200, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49f3ad98-7f18-4320-8225-e8ecbfd6782a\", \"content-length\": \"26\", \"date\": \"Fri, 30 Jan 2026 03:42:03 GMT\"} })"}
{"msg":"we are in switch slot 0","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:42:03.260048001Z","hostname":"oxz_switch","pid":2299}
Jan 30 03:42:14.778 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:42:14.778 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 20.004093378s, count: 0, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:42:14.981 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 684, tv_nsec: 318539218 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:42:14.981 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:42:14.981 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:42:29.939 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:42:29.939 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 35.208232405s, count: 1, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:42:30.465 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 699, tv_nsec: 847571466 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:42:30.466 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:42:30.466 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:42:45.519 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:42:45.613 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 50.922128743s, count: 2, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:42:46.666 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 716, tv_nsec: 85429219 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:42:46.727 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:42:46.730 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:43:01.730 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:43:01.730 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 67.039919938s, count: 3, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:43:02.924 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 732, tv_nsec: 345697439 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:43:02.943 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:43:02.943 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:43:17.944 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:43:17.944 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 83.253694606s, count: 4, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:43:23.474 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 752, tv_nsec: 895663402 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:43:23.475 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:43:23.475 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:43:38.477 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:43:38.477 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 103.786827026s, count: 5, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:43:43.606 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 773, tv_nsec: 27904583 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:43:43.607 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:43:43.607 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:43:58.785 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:43:58.801 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 124.109296424s, count: 6, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
Jan 30 03:44:12.145 DEBG lookup_socket_v6 srv, response: SrvLookup(Lookup { query: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("0aaeeb44-3717-4e23-be0e-3d441241317d.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("2c3ac73a-dc55-4965-83f7-f4742139a70e.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("_nexus._tcp.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal.") }) }, Record { name_labels: Name("683ef436-3a75-4e19-99e3-69691873ddfb.host.control-plane.oxide.internal."), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 801, tv_nsec: 561797565 } }), dns_name: _nexus._tcp.control-plane.oxide.internal, component: internal-dns-resolver
Jan 30 03:44:12.147 DEBG will register with Nexus at [fd00:1122:3344:101::c]:12221, component: producer-registration-task
Jan 30 03:44:12.148 DEBG client request, body: Some(Body), uri: http://[fd00:1122:3344:101::c]:12221/metrics/producers, method: POST, component: producer-registration-task
Jan 30 03:44:27.150 DEBG client response, result: Err(reqwest::Error { kind: Request, url: "http://[fd00:1122:3344:101::c]:12221/metrics/producers", source: TimedOut }), component: producer-registration-task
Jan 30 03:44:27.151 WARN failed to register with Nexus, will retry, error: "Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)", delay: 152.460590616s, count: 7, component: producer-registration-task, file: /home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/79b29e0/oximeter/producer/src/lib.rs:371
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:56.577887627Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","remote_addr":"[fd00:1122:3344:101::a]:39373"}
{"msg":"determined request API version","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:44:56.586571019Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"b26a6205-860f-4b32-8141-0499a985b87d","remote_addr":"[fd00:1122:3344:101::a]:39373","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","version":"3.0.0"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:56.590390457Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"b26a6205-860f-4b32-8141-0499a985b87d","remote_addr":"[fd00:1122:3344:101::a]:39373","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","latency_us":7093,"response_code":200}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:58.298735852Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","remote_addr":"[fd00:1122:3344:101::b]:59962"}
{"msg":"determined request API version","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:44:58.299009259Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"6aefc800-9a59-4dd4-94d8-3930f2c7db4f","remote_addr":"[fd00:1122:3344:101::b]:59962","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","version":"3.0.0"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:58.299169763Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"6aefc800-9a59-4dd4-94d8-3930f2c7db4f","remote_addr":"[fd00:1122:3344:101::b]:59962","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","latency_us":74,"response_code":200}
{"msg":"accepted connection","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:58.423500841Z","hostname":"oxz_switch","pid":2299,"local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","remote_addr":"[fd00:1122:3344:101::c]:48523"}
{"msg":"determined request API version","v":0,"name":"slog-rs","level":20,"time":"2026-01-30T03:44:58.425615506Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"d9eb90dd-fb65-4c81-8779-24ff295af277","remote_addr":"[fd00:1122:3344:101::c]:48523","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","version":"3.0.0"}
{"msg":"request completed","v":0,"name":"slog-rs","level":30,"time":"2026-01-30T03:44:58.425886153Z","hostname":"oxz_switch","pid":2299,"uri":"/bfd/peers","method":"GET","req_id":"d9eb90dd-fb65-4c81-8779-24ff295af277","remote_addr":"[fd00:1122:3344:101::c]:48523","local_addr":"[::]:4676","unit":"api_server","module":"admin","component":"mgd","latency_us":59,"response_code":200}