[ Mar 4 04:26:52 Enabled. ] [ Mar 4 04:26:52 Rereading configuration. ] [ Mar 4 04:26:53 Rereading configuration. ] [ Mar 4 04:27:04 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Mar 4 04:27:04 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.844755708Z","hostname":"oxz_switch","pid":4934,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-f20f786e","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.84693146Z","hostname":"oxz_switch","pid":4934} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.847359918Z","hostname":"oxz_switch","pid":4934} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.847761546Z","hostname":"oxz_switch","pid":4934,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.849612802Z","hostname":"oxz_switch","pid":4934,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n e06ca9f1-8b26-4115-81e8-dbbeca7c7ca4,\n ),\n sled_id: Some(\n d5091d4f-d9bd-4734-a00f-92d478f2bb71,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.850839165Z","hostname":"oxz_switch","pid":4934} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.850866346Z","hostname":"oxz_switch","pid":4934,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:04.858599135Z","hostname":"oxz_switch","pid":4934,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::602b:caff:fe87:a0d->62:2b:ca:87:0a:0d"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:04.861847478Z","hostname":"oxz_switch","pid":4934} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:04.868305162Z","hostname":"oxz_switch","pid":4934} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:27:04.874285198Z","hostname":"oxz_switch","pid":4934} [ Mar 4 04:27:05 Rereading configuration. ] [ Mar 4 04:27:05 Executing refresh method (:kill). ] [ Mar 4 04:27:05 Stopping because all processes in service exited. ] [ Mar 4 04:27:05 Executing stop method (:kill). ] [ Mar 4 04:27:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Mar 4 04:27:05 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.53766158Z","hostname":"oxz_switch","pid":5204,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-f20f786e","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.538662559Z","hostname":"oxz_switch","pid":5204} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.539011276Z","hostname":"oxz_switch","pid":5204} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.539131578Z","hostname":"oxz_switch","pid":5204,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.540947713Z","hostname":"oxz_switch","pid":5204,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.541037335Z","hostname":"oxz_switch","pid":5204,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: e06ca9f1-8b26-4115-81e8-dbbeca7c7ca4, sled_id: d5091d4f-d9bd-4734-a00f-92d478f2bb71, model: \"i86pc\", revision: 0, serial: \"sock\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, lot: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: 905e9a95-138e-40fd-9dd0-93c97f8b6f5f, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.541078636Z","hostname":"oxz_switch","pid":5204,"unit":"oximeter","producer_id":"a53b2e2b-4115-46dc-bc37-daa3cd05a88a"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n e06ca9f1-8b26-4115-81e8-dbbeca7c7ca4,\n ),\n sled_id: Some(\n d5091d4f-d9bd-4734-a00f-92d478f2bb71,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.544353719Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.549504598Z","hostname":"oxz_switch","pid":5204} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.550192981Z","hostname":"oxz_switch","pid":5204,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::602b:caff:fe87:a0d->62:2b:ca:87:0a:0d"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.551942685Z","hostname":"oxz_switch","pid":5204,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.552749551Z","hostname":"oxz_switch","pid":5204,"local_addr":"[fd00:1122:3344:101::2]:56410","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.552824732Z","hostname":"oxz_switch","pid":5204,"local_addr":"[fd00:1122:3344:101::2]:56410","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.552997425Z","hostname":"oxz_switch","pid":5204,"local_addr":"[fd00:1122:3344:101::2]:56410","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:56410","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.553026616Z","hostname":"oxz_switch","pid":5204,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.553045546Z","hostname":"oxz_switch","pid":5204,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.55377736Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.55427698Z","hostname":"oxz_switch","pid":5204,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.554339431Z","hostname":"oxz_switch","pid":5204,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:56410","producer_id":"a53b2e2b-4115-46dc-bc37-daa3cd05a88a"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:05.554365932Z","hostname":"oxz_switch","pid":5204,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.554373062Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.555858661Z","hostname":"oxz_switch","pid":5204} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.556382751Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.558171075Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-03-04T04:26:48.664877145Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.558194576Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","id":"5373603740652761558"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:05.558217256Z","hostname":"oxz_switch","pid":5204,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:27:05.559669234Z","hostname":"oxz_switch","pid":5204} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:06.564368149Z","hostname":"oxz_switch","pid":5204,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::3cc4:20ff:fe91:ae61->3e:c4:20:91:ae:61"} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:06.850574808Z","hostname":"oxz_switch","pid":5204,"interface":"tfportqsfp0_0","entry":"10.151.2.174->f6:98:21:72:83:27"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.941189526Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.946415687Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.949484056Z","hostname":"oxz_switch","pid":5204} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.949658439Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.950449905Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-03-04T04:26:48.664745083Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.950503066Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","id":"7877350563179435363"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:06.950512566Z","hostname":"oxz_switch","pid":5204,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.221428483Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.226742496Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.229520799Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.774424198Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.779668359Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:08.782256119Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:09.460271744Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:09.463884053Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:09.466527204Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.103333394Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.107667048Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.110601014Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.557752417Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6634, tv_nsec: 528635464 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.558365389Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.600271747Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.665360363Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.670047763Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:10.673208684Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:11.366920881Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:11.371884107Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:11.374538338Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:12.189040605Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:12.194354797Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:12.197707172Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:13.551782054Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:13.556607307Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:13.559754278Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:14.170340882Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:14.174994442Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:14.177971849Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:15.402377311Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:15.406723875Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:15.409270064Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:15.563627651Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:16.53747412Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:16.542732852Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:16.545477725Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:16.952869891Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:17.757491317Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:17.761887162Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:17.764493792Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:19.109791254Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:19.11475049Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:19.117949192Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:20.14100362Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:20.145642379Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:20.148843621Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:22.045095416Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:22.050258556Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:22.053434608Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:23.510074343Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:23.515211962Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:23.518251611Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.168791331Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.173053163Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.176327017Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.791161255Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.795636452Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:24.79865915Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:25.778749467Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:25.814913571Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:27:25.816077553Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"20.049048536s","count":"0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.013232626Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.017906496Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.020423125Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.090557072Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6649, tv_nsec: 849641654 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.091125253Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:26.123682753Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:27.164763842Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:27.318400363Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:27.323265277Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:27.326220734Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:28.268416855Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:28.27900065Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:28.28726127Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:29.673222162Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:29.679068445Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:29.682288727Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:30.967858477Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:30.998765754Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:31.004967054Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:32.258193327Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:32.269080247Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:32.272521984Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:33.608350043Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:33.613594534Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:33.61646903Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:34.846873721Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:34.851437919Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:34.854013968Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:35.753567142Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:36.001486446Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:36.006521763Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:36.009228956Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:37.140284556Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:37.187998348Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:37.193138118Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:37.196800439Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:38.338390943Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:38.342986612Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:38.346198354Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:39.299306943Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:39.30583095Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:39.309342348Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:44.645028582Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:44.649473848Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:27:44.658728487Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"38.943644459s","count":"1"} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:44.659226637Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:44.664737363Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.62977325Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.637800241Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.64386972Z","hostname":"oxz_switch","pid":5204} {"msg":"removed stale NDP entry :fe80::3cc4:20ff:fe91:ae61->3e:c4:20:91:ae:61","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:45.662790508Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.80857452Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.84388202Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6669, tv_nsec: 640955200 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:45.859212367Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:46.173285489Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:46.59876158Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:46.605836297Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:46.609950476Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:47.122036169Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-04T04:27:47.332989956Z","hostname":"oxz_switch","pid":5204,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::3cc4:20ff:fe91:ae61->3e:c4:20:91:ae:61"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:47.805222556Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:47.818287555Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:47.826874141Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:48.922794441Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:48.930496329Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:48.934548307Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:50.371991372Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:50.380040899Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:50.384709857Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:51.325446472Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:51.331150657Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:51.333768632Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:52.668631214Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:52.675882658Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:52.680366927Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:53.455713947Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:53.461390295Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:53.464575445Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:54.235042718Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:54.239938643Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:54.242751817Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:55.339822659Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:55.344745174Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:55.347894265Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:55.81800235Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:56.135311008Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:56.1400903Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:56.142844625Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:57.130055697Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:57.42254688Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:57.42773988Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:57.430526513Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:58.268764227Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:58.273782624Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:58.276886974Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:59.553556852Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:59.559021097Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:27:59.562523645Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:00.498698247Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:00.504097601Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:00.507516577Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.18443916Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:28:01.186440478Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"55.466276681s","count":"2"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.337723956Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.344105839Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.34729984Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.797261967Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6685, tv_nsec: 601814699 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.797745127Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.831738552Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.966686175Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.971719932Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:01.974666678Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:02.723332025Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:02.727637308Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:02.731048534Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:04.150715089Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:04.15542826Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:04.158108972Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:05.274579941Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:05.278574538Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:05.281376992Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:05.820715482Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:06.460261665Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:06.465226461Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:06.467640457Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.131870647Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.441061379Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.44634543Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.449046873Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.979091944Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.983775085Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:07.986436396Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.295206584Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.300091048Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.30277346Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.851260737Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.856014059Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:09.859342423Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.091852161Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.097150773Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.100406826Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.749680266Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.755098041Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:11.757918845Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:12.332165958Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:12.337198355Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:12.340092481Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:13.70281273Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:13.708116622Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:13.711047898Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:15.070507254Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:15.075302397Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:15.078427597Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:15.822266161Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:16.05971539Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:16.064703457Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:16.067514521Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:16.832743427Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:28:16.834036102Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"71.113623095s","count":"3"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:17.133242851Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:17.200644141Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:17.205801791Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:17.208812229Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:18.571293963Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:18.576735748Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:18.579664075Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.01344665Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6702, tv_nsec: 818358873 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.01393734Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.047293493Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.544036041Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.550037137Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:19.553187918Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:20.886428408Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:20.8911646Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:20.893803381Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:22.281870178Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:22.28769817Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:22.290521944Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:23.477111646Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:23.481841018Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:23.485318655Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.023747908Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.028892427Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.032316333Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.737611314Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.743892765Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:24.747352762Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:25.823900122Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:26.113626019Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:26.119155286Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:26.12198949Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.135066145Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.266746155Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.272546337Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.275466403Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.971429634Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.976032383Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:27.979137413Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:28.910567535Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:28.916201103Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:28.919176471Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:29.763497923Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:29.768803485Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:29.771486786Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:30.618255075Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:30.62422285Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:30.627125236Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:31.282446813Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:31.288086952Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:31.291306614Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:32.20327542Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:32.208302527Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:32.211253324Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:33.675753686Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:33.681311233Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:33.684436753Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:34.049076265Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:28:34.050189496Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"88.329653609s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:34.638266526Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:34.644241202Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:34.647374932Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:35.825575522Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.053921025Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.059210696Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.062052112Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.692435538Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.698706319Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:36.70188056Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.136091723Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.678169107Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.68353419Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.686221832Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.7514407Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6721, tv_nsec: 556172323 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.75196046Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:37.784897665Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:38.713084444Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:38.71910784Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:38.722393533Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:39.471349876Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:39.476364703Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:39.479272719Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:40.76740734Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:40.772962557Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:40.776123238Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:42.24795796Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:42.253755742Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:42.256606077Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:43.578515979Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:43.583585467Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:43.586688886Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:44.445779333Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:44.451251419Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:44.454477151Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:45.208584743Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:45.214221621Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:45.217316381Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:45.827904806Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:46.447223788Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:46.452192584Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:46.455404846Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:47.137499119Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:47.6664414Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:47.671873224Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:47.674592637Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:48.677448765Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:48.683219727Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:48.686359917Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:50.04928302Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:50.05496253Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:50.057779854Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:51.437897967Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:51.443443654Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:51.446562564Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:52.393066867Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:52.399356788Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:52.402864956Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:52.786371961Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:28:52.787157357Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"107.066454709s","count":"5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:53.65075713Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:53.655966481Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:53.659068541Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:54.329448528Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:54.334908763Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:54.338086124Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.165081552Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.170055448Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.172976014Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.788978793Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.793829397Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.796198553Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:55.829638727Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:57.139444255Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:57.210977135Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:57.216728016Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:57.21952729Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:58.603228573Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:58.608375762Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:58.611435961Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:59.85177696Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:59.856845018Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:28:59.85955057Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:00.971901631Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:00.976583261Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:00.978863285Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.517732006Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.523320584Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.526447015Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.74194934Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6745, tv_nsec: 546542213 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.742500661Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:01.774833365Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:02.396250938Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:02.401613271Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:02.404251912Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:03.346804188Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:03.351106471Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:03.353997877Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:04.340161194Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:04.34561697Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:04.348612847Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:05.801901533Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:05.807328668Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:05.809941788Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:05.831057586Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:06.629698796Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:06.635643131Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:06.638876173Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:07.140400744Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:08.09739569Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:08.102144601Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:08.105245231Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:09.401350744Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:09.406885591Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:09.410045642Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:10.076209428Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:10.082489559Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:10.085387205Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:11.23743433Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:11.243178101Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:11.246489145Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:12.720342597Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:12.725927124Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:12.729060755Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:13.929092426Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:13.934657053Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:13.937820804Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:15.28749541Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:15.293392174Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:15.296763869Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:15.832234146Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:16.611932781Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:16.617831125Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:16.62123476Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:16.775875383Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:29:16.776833501Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"131.055995604s","count":"6"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:17.141662536Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:17.857248896Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:17.863467376Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:17.865991464Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:18.559293034Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:18.564231329Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:18.567525352Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:19.236254481Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:19.24134601Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:19.244080243Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:20.229916315Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:20.235622045Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:20.238148504Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:21.539661213Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:21.545189599Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:21.547763479Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:22.702547548Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:22.707959493Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:22.710879879Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.245021249Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.250091027Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.25289645Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.926000401Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.930473988Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:23.933254251Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:25.089076897Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:25.094970853Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:25.097798222Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:25.833848985Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:26.080235555Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:26.084951676Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:26.087893053Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:27.143287634Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:27.380777554Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:27.385503805Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:27.388548074Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:28.069114578Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:28.073086375Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:28.075823458Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:29.255543336Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:29.261038633Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:29.265277944Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:30.596472845Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:30.605363086Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:30.611074826Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:31.201570702Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:31.209608968Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:31.214845589Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.367062156Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.3729585Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.378082739Z","hostname":"oxz_switch","pid":5204} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.903344928Z","hostname":"oxz_switch","pid":5204,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"4ede623f-0a9b-4673-a440-86c711e915b3.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"66a78d6a-f7fb-49a2-93db-2a30278f1c9b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"c27aa8c9-841e-419c-a050-6374fadac79b.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::a)) }], valid_until: Instant { tv_sec: 6776, tv_nsec: 707111744 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.90602749Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:32.974142023Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:33.166986601Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:33.183122883Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:33.190728309Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:40.037603517Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:40.040926621Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:40.05852737Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:40.0834173Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:40.122050775Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:41.584935125Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:41.592274227Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:41.596921446Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:42.787832642Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:42.795087232Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:42.799976246Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:43.930235992Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:43.9353096Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:43.938321958Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:45.183219544Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:45.188736041Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:45.191350951Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:46.56964338Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:46.576102884Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:46.580336616Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:47.702495435Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:47.71314214Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:47.722864088Z","hostname":"oxz_switch","pid":5204} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:47.984026634Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-04T04:29:48.095514164Z","hostname":"oxz_switch","pid":5204,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"162.374792796s","count":"7"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:48.95930556Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:48.965440488Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:48.969583218Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:50.041467257Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:50.043004167Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:50.304767084Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:50.31024262Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:50.313663356Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:51.153575683Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:51.158520268Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:51.161125539Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:52.167801442Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:52.17341504Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:52.176686414Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:53.552858042Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:53.558671544Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:53.561732033Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:54.98346257Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:54.989221352Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:54.992346702Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:55.86033108Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:55.865080731Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:55.86809122Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:56.642701348Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:56.650389957Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:56.653894414Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:58.055320881Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:58.062639212Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:58.068015336Z","hostname":"oxz_switch","pid":5204} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:59.539681088Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:59.545217954Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:29:59.548309094Z","hostname":"oxz_switch","pid":5204} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:30:00.043216558Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"5373603740652761558"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:30:00.050747593Z","hostname":"oxz_switch","pid":5204,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"7877350563179435363"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:30:00.736883805Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportrear0_0 addrs [fe80::602b:caff:fe87:a0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:30:00.742651816Z","hostname":"oxz_switch","pid":5204} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-04T04:30:00.746179974Z","hostname":"oxz_switch","pid":5204}