[ Jun 22 22:13:52 Enabled. ] [ Jun 22 22:13:52 Rereading configuration. ] [ Jun 22 22:13:53 Rereading configuration. ] [ Jun 22 22:14:04 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Jun 22 22:14:04 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.845231487Z","hostname":"oxz_switch","pid":5360,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-cc0c307c-dirty","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.847561386Z","hostname":"oxz_switch","pid":5360} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.847949844Z","hostname":"oxz_switch","pid":5360} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.850128949Z","hostname":"oxz_switch","pid":5360,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.850400745Z","hostname":"oxz_switch","pid":5360,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 84f084d9-433f-4517-b608-f41abe585fed,\n ),\n sled_id: Some(\n c5e4938a-fadf-4b77-8fc1-0d727fea351e,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.851655661Z","hostname":"oxz_switch","pid":5360} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.851685161Z","hostname":"oxz_switch","pid":5360,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:04.856223826Z","hostname":"oxz_switch","pid":5360,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::700e:1ff:fe58:44df->72:0e:01:58:44:df"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:04.858595845Z","hostname":"oxz_switch","pid":5360} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:04.864170732Z","hostname":"oxz_switch","pid":5360} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:04.869894971Z","hostname":"oxz_switch","pid":5360} [ Jun 22 22:14:05 Rereading configuration. ] [ Jun 22 22:14:05 Executing refresh method (:kill). ] [ Jun 22 22:14:05 Stopping because all processes in service exited. ] [ Jun 22 22:14:05 Executing stop method (:kill). ] [ Jun 22 22:14:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Jun 22 22:14:05 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.485800619Z","hostname":"oxz_switch","pid":5442,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-cc0c307c-dirty","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.486890271Z","hostname":"oxz_switch","pid":5442} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.487338761Z","hostname":"oxz_switch","pid":5442} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.487455093Z","hostname":"oxz_switch","pid":5442,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.489360773Z","hostname":"oxz_switch","pid":5442,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.489592728Z","hostname":"oxz_switch","pid":5442,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: 84f084d9-433f-4517-b608-f41abe585fed, sled_id: c5e4938a-fadf-4b77-8fc1-0d727fea351e, model: \"i86pc\", revision: 0, serial: \"buskin\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, fuse: Some(FuseData { chip_rev: ChipRevision { device_id: 272, rev: \"B1\", rev_num: 2 }, disabled: DisabledFeatures { bsync: false, mau: [0, 0, 0, 0], pgen: false, pipes: 0, ports: 0, resub: false, speeds: 0, tm_mem: 0 }, frequency: FrequencySettings { bps: 3, bps_ext: 0, cpu_speed_dis: 0, disabled: false, pcie_dis: 0, pps: 3, pps_ext: 0 }, manufacturing: ManufacturingData { core_repair: false, die_rotation: false, pmro_and_skew: 0, silent_spin: 0, soft_pipe_dis: 0, tile_repair: false, voltage_scaling: 0, wf_core_repair: false }, part: PartInfo { part_num: 4660, pkg_id: 1, version: 2 } }), lot: None, lotnum: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: 8bd2512d-9baf-4e91-bedb-7c55e9cc3264, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.489660089Z","hostname":"oxz_switch","pid":5442,"unit":"oximeter","producer_id":"cac6467e-224f-40f1-8712-853d287e5897"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 84f084d9-433f-4517-b608-f41abe585fed,\n ),\n sled_id: Some(\n c5e4938a-fadf-4b77-8fc1-0d727fea351e,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.490582718Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.499392342Z","hostname":"oxz_switch","pid":5442} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.500902824Z","hostname":"oxz_switch","pid":5442,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::700e:1ff:fe58:44df->72:0e:01:58:44:df"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.501108988Z","hostname":"oxz_switch","pid":5442,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::10ec:45ff:fe24:12f5->12:ec:45:24:12:f5"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.502320933Z","hostname":"oxz_switch","pid":5442,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.503472547Z","hostname":"oxz_switch","pid":5442,"local_addr":"[fd00:1122:3344:101::2]:51751","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.503552239Z","hostname":"oxz_switch","pid":5442,"local_addr":"[fd00:1122:3344:101::2]:51751","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.503663471Z","hostname":"oxz_switch","pid":5442,"local_addr":"[fd00:1122:3344:101::2]:51751","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:51751","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.503687482Z","hostname":"oxz_switch","pid":5442,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.503695162Z","hostname":"oxz_switch","pid":5442,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.503894846Z","hostname":"oxz_switch","pid":5442,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.504234453Z","hostname":"oxz_switch","pid":5442,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.504256613Z","hostname":"oxz_switch","pid":5442,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:51751","producer_id":"cac6467e-224f-40f1-8712-853d287e5897"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:05.504269664Z","hostname":"oxz_switch","pid":5442,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.504276334Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.504880596Z","hostname":"oxz_switch","pid":5442} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.505314995Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:05.505395747Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"proto error: no records found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"1.123264ms","count":"0"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.506694214Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-06-22T22:13:48.583459806Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.506718105Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","id":"10893021871167817936"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:05.506729515Z","hostname":"oxz_switch","pid":5442,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:05.507974081Z","hostname":"oxz_switch","pid":5442} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:05.63264725Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"proto error: no records found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"128.358476ms","count":"1"} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2026-06-22T22:14:06.303598765Z","hostname":"oxz_switch","pid":5442,"interface":"tfportqsfp0_0","entry":"10.151.1.174->5e:8e:6a:ef:c5:ee"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:06.332453736Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"proto error: no records found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"828.135342ms","count":"2"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.707537885Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.71255816Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.715545762Z","hostname":"oxz_switch","pid":5442} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.715786007Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.716592284Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-06-22T22:13:48.583329793Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.716649135Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","id":"17889303807894568400"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:06.716665095Z","hostname":"oxz_switch","pid":5442,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.481206151Z","hostname":"oxz_switch","pid":5442,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"8e6df471-5a81-4b5c-9644-274bf456a1e6.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c827f6cc-2912-4ecf-ad68-944b0eed86ea.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 708, tv_nsec: 184101500 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.481723952Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.500884111Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.801349144Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.806625254Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:07.809409432Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.417862045Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.423532913Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.427153928Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.97943362Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.984344352Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:08.987378945Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:09.539068975Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:09.545463188Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:09.548317908Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.056083406Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.060665121Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.063413398Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.941155054Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.94524987Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:11.948278233Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:12.553912936Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:12.558748267Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:12.561481864Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:13.721088115Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:13.726479778Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:13.729626673Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:14.574220518Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:14.578565319Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:14.581600492Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:15.50939114Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10893021871167817936"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:15.734525713Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:15.738556127Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:15.742271345Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:16.676738953Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:16.681689656Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:16.685055766Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:16.718220887Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"17889303807894568400"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.183855437Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.189016695Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.192776263Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.876347531Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.881675783Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:18.885007252Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:20.063009846Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:20.067919058Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:20.071033833Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:21.489236805Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:21.493561635Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:21.496966866Z","hostname":"oxz_switch","pid":5442} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:22.502156508Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:22.503136628Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"16.998847665s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:22.67448733Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:22.678160527Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:22.685421618Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:23.297971336Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:23.302260535Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:23.305955472Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:23.997249401Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:24.000895817Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:24.003559803Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.383142556Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.387721832Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.390926809Z","hostname":"oxz_switch","pid":5442} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.464821789Z","hostname":"oxz_switch","pid":5442,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"8e6df471-5a81-4b5c-9644-274bf456a1e6.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c827f6cc-2912-4ecf-ad68-944b0eed86ea.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 725, tv_nsec: 952725039 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.465666696Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:25.472117751Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:26.032738482Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10893021871167817936"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:26.30175483Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:26.306009979Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:26.308388148Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:27.495892626Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"17889303807894568400"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.004699442Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.009280938Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.011859611Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.591832004Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.597239146Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:28.600146477Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:29.279115998Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:29.282838016Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:29.28592578Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:30.530083404Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:30.534758252Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:30.538095402Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:31.310773081Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:31.315235154Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:31.318026264Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:32.73700676Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:32.741383711Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:32.744512307Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:33.689559993Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:33.694567619Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:33.697542169Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:34.666760403Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:34.676528667Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:34.679688283Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:35.222659222Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:35.227072735Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:35.230391354Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:36.266615054Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10893021871167817936"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:36.385696333Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:36.390545324Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:36.393544247Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.152206403Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.156623465Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.159863723Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.472825123Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"17889303807894568400"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.895496947Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.900498461Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:37.903936803Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:38.762662481Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:38.767098983Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:38.77032269Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.22431709Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.229294094Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.23245331Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.987032841Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.991621947Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.995573839Z","hostname":"oxz_switch","pid":5442} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:40.998880028Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:14:41.000005502Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"34.755002904s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:42.378270289Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:42.383428397Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:42.386503841Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:43.513503366Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:43.517955979Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:43.520678526Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:44.12372827Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:44.129733206Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:44.133033025Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:45.157565908Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:45.162344896Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:45.165518055Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.075254699Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.079447507Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.082158904Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.241589066Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10893021871167817936"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.253674198Z","hostname":"oxz_switch","pid":5442,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"8e6df471-5a81-4b5c-9644-274bf456a1e6.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c827f6cc-2912-4ecf-ad68-944b0eed86ea.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"dcbbdf34-0dce-4894-b25e-c53fc696f30a.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 746, tv_nsec: 230232094 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.254454655Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:46.262398241Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:47.385938223Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:47.391293658Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:47.394223007Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:47.44795146Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"17889303807894568400"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:48.064031226Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:48.068829556Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:48.071349479Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.093975173Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.098858135Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.102157534Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.664393185Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.669133624Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:49.672469574Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:50.718268201Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:50.724246416Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:50.727514524Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:51.564455337Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:51.569553183Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:51.573047147Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:52.953516589Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:52.958403041Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:52.961832453Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:54.237847322Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:54.243471829Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:54.246600575Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:55.180150206Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:55.185063819Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:55.18815405Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:56.216768922Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10893021871167817936"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:56.610221726Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:56.614405283Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:56.617445677Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:57.329298445Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:57.334160237Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:57.337395064Z","hostname":"oxz_switch","pid":5442} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:57.4228509Z","hostname":"oxz_switch","pid":5442,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"17889303807894568400"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.008854538Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.013676069Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.016675881Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.912924643Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.917650482Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:14:58.92135919Z","hostname":"oxz_switch","pid":5442} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:15:00.214085198Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportrear0_0 addrs [fe80::700e:1ff:fe58:44df]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:15:00.219241146Z","hostname":"oxz_switch","pid":5442} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:15:00.222850122Z","hostname":"oxz_switch","pid":5442} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-06-22T22:15:01.223092067Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-06-22T22:15:01.223902574Z","hostname":"oxz_switch","pid":5442,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"55.033849166s","count":"5"}