[ Jan 6 15:57:47 Enabled. ] [ Jan 6 15:57:47 Rereading configuration. ] [ Jan 6 15:57:48 Rereading configuration. ] [ Jan 6 15:57:59 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Jan 6 15:57:59 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.785214377Z","hostname":"oxz_switch","pid":3641,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-c9bf538d","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.829341528Z","hostname":"oxz_switch","pid":3641} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.829748185Z","hostname":"oxz_switch","pid":3641} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.83158821Z","hostname":"oxz_switch","pid":3641,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.831727503Z","hostname":"oxz_switch","pid":3641,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 35aafa39-02a1-499b-a310-0b599d539839,\n ),\n sled_id: Some(\n ea50ffdd-2d66-4d40-a211-51b9d5ff84e5,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.832964326Z","hostname":"oxz_switch","pid":3641} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.833009157Z","hostname":"oxz_switch","pid":3641,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:57:59.837501311Z","hostname":"oxz_switch","pid":3641,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::347d:deff:feb7:6e4a->36:7d:de:b7:6e:4a"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:57:59.8400655Z","hostname":"oxz_switch","pid":3641} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:57:59.845328909Z","hostname":"oxz_switch","pid":3641} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:57:59.849874184Z","hostname":"oxz_switch","pid":3641} [ Jan 6 15:58:00 Rereading configuration. ] [ Jan 6 15:58:00 Executing refresh method (:kill). ] [ Jan 6 15:58:00 Stopping because all processes in service exited. ] [ Jan 6 15:58:00 Executing stop method (:kill). ] [ Jan 6 15:58:00 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Jan 6 15:58:00 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.394259205Z","hostname":"oxz_switch","pid":3883,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-c9bf538d","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.435498101Z","hostname":"oxz_switch","pid":3883} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.435768676Z","hostname":"oxz_switch","pid":3883} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.435865808Z","hostname":"oxz_switch","pid":3883,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.437748854Z","hostname":"oxz_switch","pid":3883,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.437895716Z","hostname":"oxz_switch","pid":3883,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: 35aafa39-02a1-499b-a310-0b599d539839, sled_id: ea50ffdd-2d66-4d40-a211-51b9d5ff84e5, model: \"i86pc\", revision: 0, serial: \"buskin\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, lot: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: 4e2f4a0c-392c-4bfa-b143-1c1cf971481b, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.437923487Z","hostname":"oxz_switch","pid":3883,"unit":"oximeter","producer_id":"91d54fe8-16eb-4f21-b042-46fcb1dd54c7"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 35aafa39-02a1-499b-a310-0b599d539839,\n ),\n sled_id: Some(\n ea50ffdd-2d66-4d40-a211-51b9d5ff84e5,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.438981397Z","hostname":"oxz_switch","pid":3883} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.447444796Z","hostname":"oxz_switch","pid":3883,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::bc10:d8ff:feee:d061->be:10:d8:ee:d0:61"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.447506217Z","hostname":"oxz_switch","pid":3883} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.448304132Z","hostname":"oxz_switch","pid":3883,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::347d:deff:feb7:6e4a->36:7d:de:b7:6e:4a"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.449587637Z","hostname":"oxz_switch","pid":3883,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.450455683Z","hostname":"oxz_switch","pid":3883,"local_addr":"[fd00:1122:3344:101::2]:50479","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.450576765Z","hostname":"oxz_switch","pid":3883,"local_addr":"[fd00:1122:3344:101::2]:50479","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.450624396Z","hostname":"oxz_switch","pid":3883,"local_addr":"[fd00:1122:3344:101::2]:50479","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:50479","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.450655037Z","hostname":"oxz_switch","pid":3883,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.450673407Z","hostname":"oxz_switch","pid":3883,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.45137395Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.451791028Z","hostname":"oxz_switch","pid":3883,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.45187164Z","hostname":"oxz_switch","pid":3883,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:50479","producer_id":"91d54fe8-16eb-4f21-b042-46fcb1dd54c7"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:00.45188681Z","hostname":"oxz_switch","pid":3883,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.45189611Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.453362278Z","hostname":"oxz_switch","pid":3883} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.453819436Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.455011899Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-01-06T15:57:43.757864212Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.45504871Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","id":"7285749111225692396"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:00.45505864Z","hostname":"oxz_switch","pid":3883,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:58:00.456539478Z","hostname":"oxz_switch","pid":3883} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2026-01-06T15:58:01.357276939Z","hostname":"oxz_switch","pid":3883,"interface":"tfportqsfp0_0","entry":"10.151.1.174->92:c8:c5:f1:e0:40"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.508636369Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.513544871Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.516242322Z","hostname":"oxz_switch","pid":3883} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.516414295Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.517440735Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-01-06T15:57:43.757731799Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.517523466Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","id":"11268877359684115463"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:01.517532896Z","hostname":"oxz_switch","pid":3883,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:02.478395039Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:02.48216324Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:02.485163026Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:03.345202891Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:03.349693776Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:03.352711673Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:04.573426228Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:04.577817141Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:04.580358729Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.332184366Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.33719397Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.340007113Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.86982954Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.874403636Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:05.877370102Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.072575997Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.077971469Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.081114478Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.868060767Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.872818536Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:07.876047387Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:09.143137566Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:09.148417666Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:09.151243359Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.002566699Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.007198586Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.009985059Z","hostname":"oxz_switch","pid":3883} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:58:10.454771914Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"proto error: no records found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"10.002862024s","count":"0"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.458345401Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.714606457Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21719, tv_nsec: 386447128 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.715478843Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:10.746466747Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:11.479428479Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:11.483607878Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:11.486363719Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:11.520074224Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:12.418215546Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:12.423289782Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:12.426150666Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:13.355144218Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:13.360090431Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:13.362620569Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:14.597347069Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:14.601593619Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:14.604246969Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:15.848363726Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:15.852939672Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:15.855717344Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:17.053455707Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:17.057508194Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:17.05997609Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:18.438231263Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:18.442737738Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:18.445624162Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:19.703312915Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:19.707698507Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:19.710469879Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:20.461682544Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.162850886Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.16729526Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.170159864Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.63887148Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.689906701Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.695087718Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:21.698143246Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.199206461Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.204809717Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.207355095Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.792200077Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.796439056Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:23.799674717Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:24.680443683Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:24.685357641Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:24.688395133Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:25.535317341Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:25.539871326Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:25.542854051Z","hostname":"oxz_switch","pid":3883} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.110314584Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:58:26.111417027Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"25.29687059s","count":"1"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.250559673Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.254774362Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.257493594Z","hostname":"oxz_switch","pid":3883} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.55737707Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21734, tv_nsec: 865403406 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.557975169Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:26.588723859Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:27.281123843Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:27.285659918Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:27.288248049Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:28.624766908Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:28.628400504Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:28.631039365Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:29.300971384Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:29.306186186Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:29.308793431Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:30.703066103Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:30.707749874Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:30.71074579Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:30.84053868Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:31.23520644Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:31.239890177Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:31.242986338Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:31.902371817Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:32.516609554Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:32.521603332Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:32.524323633Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:33.814899553Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:33.820172288Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:33.823018692Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:34.674353032Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:34.67893245Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:34.68217641Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:35.943177999Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:35.947803292Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:35.950772078Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:37.217526628Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:37.221861994Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:37.224818706Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:38.319372614Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:38.32517739Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:38.327644959Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:39.139618473Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:39.144357603Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:39.147413784Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:40.277009012Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:40.281428669Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:40.284068004Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:40.869544354Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:41.631070594Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:58:41.632301126Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"40.775529163s","count":"2"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:41.670411167Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:41.675094193Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:41.677337644Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:41.931929846Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.300823149Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.304795777Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.307967314Z","hostname":"oxz_switch","pid":3883} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.803473604Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21751, tv_nsec: 67360050 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.80440332Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:42.83425546Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:43.727600339Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:43.732275535Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:43.735235369Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.021490959Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.026602696Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.029527273Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.847103694Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.85154935Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:45.85438632Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:46.864619113Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:46.869720793Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:46.872198428Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:47.517373395Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:47.521322211Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:47.524316392Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:48.899000534Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:48.903876697Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:48.907023748Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:49.829826023Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:49.83357497Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:49.83613238Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:50.427487709Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:50.432863234Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:50.435460986Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:50.8988939Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:51.019925926Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:51.023910085Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:51.026433112Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:51.960561042Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:52.130860652Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:52.135240877Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:52.13810844Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:53.600454768Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:53.604957214Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:53.60770634Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:54.895757297Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:54.900493199Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:54.90320896Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:55.457777155Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:55.462471487Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:55.465529097Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:56.258410412Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:56.263088519Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:56.2659813Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:57.680967758Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:57.685730137Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:57.688188904Z","hostname":"oxz_switch","pid":3883} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:57.875498144Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:58:57.876749659Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"56.975949567s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:58.501903944Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:58.506485829Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:58.509352099Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:59.518339072Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:59.524070301Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:58:59.527198685Z","hostname":"oxz_switch","pid":3883} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.171725024Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21768, tv_nsec: 388474217 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.172317852Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.201597877Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.360421636Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.364777713Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.367173637Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:00.927499958Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:01.547226036Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:01.551742874Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:01.554336791Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:01.990064278Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:02.876718014Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:02.88096272Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:02.883141257Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:04.191531702Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:04.195731516Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:04.198387457Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:05.116644088Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:05.12097692Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:05.123556098Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:06.559973469Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:06.564599255Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:06.567347913Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:07.658961462Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:07.663466303Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:07.666222655Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:08.863852046Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:08.868094387Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:08.870531285Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:10.365342145Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:10.369578076Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:10.372295244Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:10.956639203Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:11.203961137Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:11.208493782Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:11.211613751Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:12.019583469Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:12.095016795Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:12.100107196Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:12.102728736Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:13.347488461Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:13.35213223Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:13.355029319Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:14.429175076Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:14.434175165Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:14.437099365Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:14.99892988Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:15.004749077Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:15.007532196Z","hostname":"oxz_switch","pid":3883} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:15.24417934Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:59:15.245287721Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"74.297350629s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:16.242214355Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:16.246865474Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:16.249792724Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:17.44109735Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:17.445118178Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:17.447781953Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:18.664557909Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:18.669160226Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:18.671963973Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:19.854645257Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:19.859265744Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:19.861971553Z","hostname":"oxz_switch","pid":3883} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:20.876736856Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21789, tv_nsec: 37355444 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:20.877328267Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:20.907577958Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:20.986320294Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.194246202Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.198718482Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.20151439Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.838868181Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.844123386Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:21.846663289Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:22.048712844Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:23.263635403Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:23.268284644Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:23.271047572Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:24.235803409Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:24.241048798Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:24.243997344Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:25.698043987Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:25.701594693Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:25.703863983Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:26.694203241Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:26.699258735Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:26.701885546Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:27.332261885Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:27.336539384Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:27.343614276Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.012735733Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.018101706Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.021030138Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.907658754Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.911592818Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:28.914350742Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:29.506809186Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:29.511096681Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:29.51386157Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:30.104229058Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:30.108863168Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:30.111518621Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:31.015990908Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:31.01953976Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:31.023714006Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:31.026398292Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.077349816Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.134843033Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.141512783Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.145461704Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.67510197Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.682095515Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:32.685536037Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:33.711461419Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:33.717060768Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:33.720045321Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:34.606273209Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:34.612049712Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:34.614877538Z","hostname":"oxz_switch","pid":3883} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:36.045241751Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:59:36.045307622Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"95.029032976s","count":"5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:36.223110538Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:36.660175132Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:36.677205551Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:37.467199876Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:37.474508994Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:37.478582892Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:38.415811585Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:38.422675516Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:38.427310488Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:39.517052864Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:39.521467308Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:39.524136437Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:40.57579259Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:40.582909675Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:40.586939086Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:41.045557397Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:41.339224796Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:41.343583295Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:41.346115677Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:42.095126591Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:42.099311893Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:42.10212527Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:42.107067591Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.412827814Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.418779086Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.422380554Z","hostname":"oxz_switch","pid":3883} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.583970206Z","hostname":"oxz_switch","pid":3883,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"29329f4d-f50f-4bf3-b2c4-2002724a91fb.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"5b426f4a-50fc-4148-8a9e-9b87e9146877.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"ca60836f-ff2f-4002-9e64-de209f4411be.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 21811, tv_nsec: 684893848 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.584436535Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:43.615670643Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:44.550287125Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:44.556270848Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:44.559190703Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:45.744172901Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:45.749364929Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:45.752278794Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:47.454914319Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:47.464428238Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:47.468969583Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:48.916184866Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:48.921903494Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:48.925843858Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:49.660948941Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:49.666649398Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:49.669618964Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:50.617482444Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:50.622486738Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:50.625114798Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:51.052854462Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7285749111225692396"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:51.868471979Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:51.872917508Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:51.87570772Z","hostname":"oxz_switch","pid":3883} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:52.110916278Z","hostname":"oxz_switch","pid":3883,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11268877359684115463"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:52.591655006Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:52.596004938Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:52.598508785Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:53.829670364Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:53.8342351Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:53.837104624Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.123328665Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.126789391Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.129322708Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.697221351Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.702418389Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:55.705703461Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.350628706Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.356257812Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.359574075Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.926644884Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.934167206Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:56.938348494Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:57.719216958Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:57.726035927Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:57.729759157Z","hostname":"oxz_switch","pid":3883} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:58.507970562Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:58.513456905Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:58.516320109Z","hostname":"oxz_switch","pid":3883} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:58.618012364Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-01-06T15:59:58.620012281Z","hostname":"oxz_switch","pid":3883,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"117.597098243s","count":"6"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:59.714224215Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportrear0_0 addrs [fe80::347d:deff:feb7:6e4a]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:59.71983019Z","hostname":"oxz_switch","pid":3883} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-01-06T15:59:59.726923744Z","hostname":"oxz_switch","pid":3883}