[ Oct 13 19:42:27 Enabled. ] [ Oct 13 19:42:27 Rereading configuration. ] [ Oct 13 19:42:27 Rereading configuration. ] [ Oct 13 19:42:38 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Oct 13 19:42:38 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.839684408Z","hostname":"oxz_switch","pid":4657,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-606c0be8","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.880845886Z","hostname":"oxz_switch","pid":4657} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.881240166Z","hostname":"oxz_switch","pid":4657} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.882552088Z","hostname":"oxz_switch","pid":4657,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.883347078Z","hostname":"oxz_switch","pid":4657,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n b2abaaec-8eba-4e95-8e4b-143b707fbb16,\n ),\n sled_id: Some(\n 54fac9f3-ab45-4846-bb1e-0a3750141ba2,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.88466725Z","hostname":"oxz_switch","pid":4657} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.88468066Z","hostname":"oxz_switch","pid":4657,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:38.889626892Z","hostname":"oxz_switch","pid":4657,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::3c8a:b5ff:fe53:e54c->3e:8a:b5:53:e5:4c"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:38.891512508Z","hostname":"oxz_switch","pid":4657} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:38.896585952Z","hostname":"oxz_switch","pid":4657} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:42:38.901869301Z","hostname":"oxz_switch","pid":4657} [ Oct 13 19:42:39 Rereading configuration. ] [ Oct 13 19:42:39 Executing refresh method (:kill). ] [ Oct 13 19:42:39 Stopping because all processes in service exited. ] [ Oct 13 19:42:39 Executing stop method (:kill). ] [ Oct 13 19:42:39 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Oct 13 19:42:39 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.440966558Z","hostname":"oxz_switch","pid":4821,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-606c0be8","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.480135647Z","hostname":"oxz_switch","pid":4821} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.480410654Z","hostname":"oxz_switch","pid":4821} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.480517607Z","hostname":"oxz_switch","pid":4821,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.482194537Z","hostname":"oxz_switch","pid":4821,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.482416253Z","hostname":"oxz_switch","pid":4821,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: b2abaaec-8eba-4e95-8e4b-143b707fbb16, sled_id: 54fac9f3-ab45-4846-bb1e-0a3750141ba2, model: \"i86pc\", revision: 0, serial: \"sock\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, lot: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: 608d65be-2e51-4971-a008-d9efa992aa00, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.482445464Z","hostname":"oxz_switch","pid":4821,"unit":"oximeter","producer_id":"86dc06a1-0bae-4fa2-955d-c7e389f90ab4"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n b2abaaec-8eba-4e95-8e4b-143b707fbb16,\n ),\n sled_id: Some(\n 54fac9f3-ab45-4846-bb1e-0a3750141ba2,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.483362236Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.491378253Z","hostname":"oxz_switch","pid":4821} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.492946311Z","hostname":"oxz_switch","pid":4821,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::3c8a:b5ff:fe53:e54c->3e:8a:b5:53:e5:4c"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.493172997Z","hostname":"oxz_switch","pid":4821,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::90c0:18ff:feaa:51e2->92:c0:18:aa:51:e2"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.49370155Z","hostname":"oxz_switch","pid":4821,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.494401627Z","hostname":"oxz_switch","pid":4821,"local_addr":"[fd00:1122:3344:101::2]:42149","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.494494549Z","hostname":"oxz_switch","pid":4821,"local_addr":"[fd00:1122:3344:101::2]:42149","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.49455195Z","hostname":"oxz_switch","pid":4821,"local_addr":"[fd00:1122:3344:101::2]:42149","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:42149","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.494561611Z","hostname":"oxz_switch","pid":4821,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.494571131Z","hostname":"oxz_switch","pid":4821,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.495175926Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.495497183Z","hostname":"oxz_switch","pid":4821,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.495575065Z","hostname":"oxz_switch","pid":4821,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:42149","producer_id":"86dc06a1-0bae-4fa2-955d-c7e389f90ab4"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:39.495587676Z","hostname":"oxz_switch","pid":4821,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.495607036Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.497208745Z","hostname":"oxz_switch","pid":4821} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.497623595Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.49782112Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2025-10-13T19:42:22.912074631Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.497868632Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","id":"1519653689806665092"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:39.497889362Z","hostname":"oxz_switch","pid":4821,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:42:39.500117967Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.504610233Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.509334249Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.512355593Z","hostname":"oxz_switch","pid":4821} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.512562538Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.513647904Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2025-10-13T19:42:22.911929588Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.513714616Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","id":"4060966782570408284"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:40.513731146Z","hostname":"oxz_switch","pid":4821,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2025-10-13T19:42:40.906790575Z","hostname":"oxz_switch","pid":4821,"interface":"tfportqsfp0_0","entry":"10.151.2.174->12:73:5e:a8:75:e0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.026820505Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.03193035Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.034759559Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.69080984Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.696466819Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:41.699051642Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:42.754555518Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:42.758898685Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:42.761483228Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:43.717832445Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:43.723911654Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:43.727138583Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.498826106Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3346, tv_nsec: 797688031 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.499324208Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.530139673Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.706990045Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.713738261Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:44.717995475Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:46.201178107Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:46.205941534Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:46.208715711Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:47.39206219Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:47.39657262Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:47.399198125Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:48.690604519Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:48.695906829Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:48.698581044Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:49.501890672Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:49.672406359Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:49.677547235Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:49.680419176Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:50.458882665Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:50.463512549Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:50.466158494Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:50.516391964Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:51.824130469Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:51.829558482Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:51.832505914Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:53.140552115Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:53.145429305Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:53.148061199Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:54.265080872Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:54.270510225Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:54.273403096Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:55.277090742Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:55.28149996Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:55.284384651Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:56.358552655Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:56.362674406Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:56.365510955Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:57.591334823Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:57.596807087Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:57.599700988Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:58.205161209Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:58.209436534Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:58.212059888Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.427981715Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.455107899Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:42:59.456379411Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"20.037773911s","count":"0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.574284149Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.582653274Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.587517813Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.792018662Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3362, tv_nsec: 167957405 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.792486404Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:42:59.827778749Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:00.441241775Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:00.544301406Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:00.549530103Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:00.552279093Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:01.841467239Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:01.847838696Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:01.850897571Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:02.853490247Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:02.859137836Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:02.862321264Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:04.369214008Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:04.38436578Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:04.391329771Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:05.03252756Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:05.037573664Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:05.040697611Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:06.435215394Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:06.439661233Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:06.442540013Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:07.345383211Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:07.35145483Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:07.354909965Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:08.472507767Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:08.476805943Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:08.479722895Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:09.070637609Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:09.075373955Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:09.077839116Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:09.40451603Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:10.033100949Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:10.039481616Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:10.042542401Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:10.416774414Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:11.494159167Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:11.499355045Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:11.502128843Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:12.479785667Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:12.485377964Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:12.488308666Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:13.194695908Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:13.201331471Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:13.204628292Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:17.597037001Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:17.59904425Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:17.604950815Z","hostname":"oxz_switch","pid":4821} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:43:17.610464151Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"38.239736021s","count":"1"} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:17.645041534Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:18.723467959Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3381, tv_nsec: 148674192 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:18.724126192Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:18.774540133Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.054736795Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.062784363Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.066796181Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.388723119Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.761730931Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.767993385Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:19.773143011Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:20.393508829Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.107935307Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.114761995Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.119340138Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.722295509Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.736274072Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:21.740324741Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:22.297245291Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:22.30578004Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:22.310065985Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:23.780953616Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:23.785025496Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:23.787599379Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.028657593Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.03382287Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.037217943Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.728290058Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.733854745Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:25.736684075Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:26.293279337Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:26.297510443Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:26.300462093Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:27.544263256Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:27.549410012Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:27.552753424Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:28.955423348Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:28.959935209Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:28.962887722Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:29.364163239Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:30.325449011Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:30.331420198Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:30.335082148Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:30.368665453Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:31.466952059Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:31.472375775Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:31.475715755Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:32.541555524Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:32.54748591Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:32.550432082Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:33.735263845Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:43:33.73750118Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"54.41070527s","count":"2"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:33.903374303Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:33.908498869Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:33.911619856Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.883227271Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3397, tv_nsec: 354053653 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.883744314Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.930661476Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.958297235Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.965274137Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:34.969535201Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:35.819362896Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:35.825059356Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:35.82806895Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:36.608597033Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:36.614004485Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:36.617087841Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:37.440100647Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:37.445355796Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:37.448610046Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:38.409752754Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:38.414177093Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:38.417230608Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:39.118492873Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:39.123609418Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:39.126183761Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:39.339512062Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:40.127382304Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:40.132526801Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:40.135193726Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:40.343839091Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:41.542998606Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:41.548104981Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:41.550846139Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:42.175722867Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:42.180798572Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:42.18399449Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.035010423Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.039553445Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.042797285Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.657813332Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.663356578Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:43.667005887Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:44.643655697Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:44.649033289Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:44.652111645Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:45.526991274Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:45.53250731Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:45.535563375Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:46.951421301Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:46.956328892Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:46.958896615Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:48.076405864Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:48.082165635Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:48.085308473Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:49.314418874Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:49.475162702Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:49.480142785Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:49.483318783Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:49.891316624Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:43:49.892454401Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"70.609616632s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:50.286592803Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:50.291533154Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:50.29421389Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:50.319234164Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.313101457Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.318037518Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.320794256Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.751483995Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3414, tv_nsec: 268228148 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.752042649Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:51.781576664Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:52.075690759Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:52.080822354Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:52.083539391Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:53.535766132Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:53.541279047Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:53.544027475Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:54.531418358Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:54.536646627Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:54.539698561Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:55.64743094Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:55.65272372Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:55.655938519Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:56.53169626Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:56.537011601Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:56.540046195Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:57.791151196Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:57.79740962Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:57.80067872Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:59.20032874Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:59.204761919Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:59.207385393Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:43:59.290468874Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:00.295393288Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:00.534231485Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:00.539446923Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:00.542853327Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:01.245374217Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:01.250552409Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:01.253616955Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.239420879Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.244850002Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.247778604Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.941535004Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.947105551Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:02.950171767Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:03.498842164Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:03.504552734Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:03.507849535Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.111507983Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.117185183Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.120273609Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.873469819Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.878940853Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:04.882167953Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:05.533594514Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:05.539172851Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:05.542262157Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:06.53533275Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:06.542804164Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:06.546500905Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:06.74174755Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:44:06.743860482Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"87.506862762s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:07.842570809Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:07.863310039Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:07.868148637Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.270412601Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.310520067Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.317147289Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.324515901Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.870303806Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.877091703Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:09.881574933Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:10.271742457Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:10.821890421Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3433, tv_nsec: 390146143 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:10.82311164Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:11.166905625Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:11.283558141Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:11.29042187Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:11.296664373Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:12.085640582Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:12.092753107Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:12.097072183Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.160088724Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.165052826Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.16787205Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.939365806Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.946915911Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:13.950558971Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.10834873Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.113277551Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.115908745Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.692720643Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.697953312Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:15.701051238Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:16.286885458Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:16.292894326Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:16.345837836Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:17.559852546Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:17.565223428Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:17.568646172Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:18.643944655Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:18.648171169Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:18.650936775Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:19.282211899Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.074595876Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.080246774Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.083485984Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.246218511Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.721670099Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.726925958Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:20.729767418Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:21.5346995Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:21.54286261Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:21.552877187Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:22.33429376Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:22.338502604Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:22.341656631Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:23.053799163Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:23.058741674Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:23.062002224Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.052172573Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.057674868Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.060827065Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.974833635Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.980536635Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:24.983958748Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:26.132924852Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:44:26.134067911Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"106.944116691s","count":"5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:26.459680054Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:26.464413822Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:26.467388875Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.270420963Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.276671605Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.279666014Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.885440736Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.892045834Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:27.895556193Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:28.469923039Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:28.475174547Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:28.478404516Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:29.273107642Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:29.848112267Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:29.85439475Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:29.859421113Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:30.241047873Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:31.309145491Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:31.314433921Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:31.318013519Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:32.34141003Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:32.351468217Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:32.361472172Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.131663369Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.138621059Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.1427359Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.953569836Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.961546131Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:33.96558715Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:35.32251165Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:35.333724245Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:35.338090901Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.441469868Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.449332501Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.454226321Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.948479267Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3459, tv_nsec: 551112099 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.949094612Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:36.984457928Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:37.196697738Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:37.202642713Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:37.205831641Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:38.578785984Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:38.586735999Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:38.589994558Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:39.275417879Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:40.095939387Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:40.103991775Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:40.107621184Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:40.243631316Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.110789968Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.116912148Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.120221979Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.785560287Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.790888137Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.794022314Z","hostname":"oxz_switch","pid":4821} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:41.898411951Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::b]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:44:41.899973049Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"122.70802671s","count":"6"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:42.946149678Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:42.951678723Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:42.954422901Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:43.548961355Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:43.554369788Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:43.557083405Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:44.540097245Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:44.544620466Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:44.547174819Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.194498756Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.199756515Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.202381089Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.930946566Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.936102363Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:45.938884531Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:47.250218175Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:47.255655298Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:47.258938578Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:48.366368917Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:48.372452855Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:48.375536441Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:49.27768891Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:49.642927037Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:49.648554095Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:49.651829795Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:50.246870711Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.056020872Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.061586479Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.065095655Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.636324068Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.641854123Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:51.644833516Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.138017163Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.143682742Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.146980222Z","hostname":"oxz_switch","pid":4821} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.677281953Z","hostname":"oxz_switch","pid":4821,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7faaf9e3-47da-4afd-9af7-cd7c28477720.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"bd343e8a-b1ad-4572-8c75-639bff9350fe.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f0102768-c4b0-4f28-9630-c703a2e4fe2b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 3476, tv_nsec: 279971455 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::b]:12221","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.677881768Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.713681635Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::b]:12221/metrics/producers","method":"POST"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:53.850806774Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Ok(Response { url: \"http://[fd00:1122:3344:101::b]:12221/metrics/producers\", status: 503, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f146de75-3353-47ee-8d1c-4b9bb27dc425\", \"content-length\": \"133\", \"date\": \"Mon, 13 Oct 2025 19:44:53 GMT\"} })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-10-13T19:44:53.85432356Z","hostname":"oxz_switch","pid":4821,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/e76962a/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f146de75-3353-47ee-8d1c-4b9bb27dc425\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Mon, 13 Oct 2025 19:44:53 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"f146de75-3353-47ee-8d1c-4b9bb27dc425\\\" }\"","delay":"134.66249457s","count":"7"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:54.180178812Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:54.185039981Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:54.188188958Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:55.552693434Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:55.558556627Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:55.561892469Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:57.018271145Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:57.024421476Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:57.027954813Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:58.373807971Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:58.379529281Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:58.382708419Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:59.280947993Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1519653689806665092"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:59.824161719Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:59.829991582Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:44:59.833516158Z","hostname":"oxz_switch","pid":4821} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:45:00.250407681Z","hostname":"oxz_switch","pid":4821,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"4060966782570408284"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:45:00.952342786Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportrear0_0 addrs [fe80::3c8a:b5ff:fe53:e54c]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:45:00.961342316Z","hostname":"oxz_switch","pid":4821} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:45:00.966243796Z","hostname":"oxz_switch","pid":4821} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-10-13T19:45:01.949707548Z","hostname":"oxz_switch","pid":4821}