[ Dec 24 00:27:56 Enabled. ] [ Dec 24 00:27:56 Rereading configuration. ] [ Dec 24 00:27:56 Rereading configuration. ] [ Dec 24 00:28:07 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Dec 24 00:28:07 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.837206225Z","hostname":"oxz_switch","pid":4774,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-c9bf538d","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.880427337Z","hostname":"oxz_switch","pid":4774} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.880713061Z","hostname":"oxz_switch","pid":4774} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.882846171Z","hostname":"oxz_switch","pid":4774,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.883031784Z","hostname":"oxz_switch","pid":4774,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 729b5b4c-a7bb-4b46-a1cb-c499f112354d,\n ),\n sled_id: Some(\n e77c997b-b935-4b95-8f26-f3fd82a83309,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.884537676Z","hostname":"oxz_switch","pid":4774} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.884571516Z","hostname":"oxz_switch","pid":4774,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:07.889458516Z","hostname":"oxz_switch","pid":4774,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::4844:10ff:fe11:93e5->4a:44:10:11:93:e5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:07.892043663Z","hostname":"oxz_switch","pid":4774} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:07.897444961Z","hostname":"oxz_switch","pid":4774} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:28:07.902515694Z","hostname":"oxz_switch","pid":4774} [ Dec 24 00:28:08 Rereading configuration. ] [ Dec 24 00:28:08 Executing refresh method (:kill). ] [ Dec 24 00:28:08 Stopping because all processes in service exited. ] [ Dec 24 00:28:08 Executing stop method (:kill). ] [ Dec 24 00:28:08 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Dec 24 00:28:08 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.47229299Z","hostname":"oxz_switch","pid":4920,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-c9bf538d","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.512538039Z","hostname":"oxz_switch","pid":4920} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.512841673Z","hostname":"oxz_switch","pid":4920} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.512913164Z","hostname":"oxz_switch","pid":4920,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.51474104Z","hostname":"oxz_switch","pid":4920,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.514854802Z","hostname":"oxz_switch","pid":4920,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: 729b5b4c-a7bb-4b46-a1cb-c499f112354d, sled_id: e77c997b-b935-4b95-8f26-f3fd82a83309, model: \"i86pc\", revision: 0, serial: \"buskin\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, lot: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: 6ece1e8a-a809-493a-bd33-9c994728fb64, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.514875022Z","hostname":"oxz_switch","pid":4920,"unit":"oximeter","producer_id":"fb29dcbd-b687-49e9-93de-abfd3492fd20"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 729b5b4c-a7bb-4b46-a1cb-c499f112354d,\n ),\n sled_id: Some(\n e77c997b-b935-4b95-8f26-f3fd82a83309,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.515965978Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.523279213Z","hostname":"oxz_switch","pid":4920} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.523940422Z","hostname":"oxz_switch","pid":4920,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::4e1:3cff:fec2:5334->06:e1:3c:c2:53:34"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.524219476Z","hostname":"oxz_switch","pid":4920,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::4844:10ff:fe11:93e5->4a:44:10:11:93:e5"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.525252701Z","hostname":"oxz_switch","pid":4920,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.52588297Z","hostname":"oxz_switch","pid":4920,"local_addr":"[fd00:1122:3344:101::2]:33932","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.525958501Z","hostname":"oxz_switch","pid":4920,"local_addr":"[fd00:1122:3344:101::2]:33932","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.526019772Z","hostname":"oxz_switch","pid":4920,"local_addr":"[fd00:1122:3344:101::2]:33932","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:33932","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.526029342Z","hostname":"oxz_switch","pid":4920,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.526039213Z","hostname":"oxz_switch","pid":4920,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.526147994Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.527075148Z","hostname":"oxz_switch","pid":4920,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.527151538Z","hostname":"oxz_switch","pid":4920,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:33932","producer_id":"fb29dcbd-b687-49e9-93de-abfd3492fd20"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:08.527170739Z","hostname":"oxz_switch","pid":4920,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.527185069Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.5286645Z","hostname":"oxz_switch","pid":4920} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.529028316Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.529206738Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2025-12-24T00:27:51.801803239Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.529243669Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","id":"10274436776871185655"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:08.529256889Z","hostname":"oxz_switch","pid":4920,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:28:08.53140843Z","hostname":"oxz_switch","pid":4920} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:28:09.105307295Z","hostname":"oxz_switch","pid":4920,"interface":"tfportqsfp0_0","entry":"10.151.1.174->92:33:d6:8e:3d:44"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.806326098Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.810916244Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.813917547Z","hostname":"oxz_switch","pid":4920} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.814200451Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.815340187Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2025-12-24T00:27:51.801670257Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.815399868Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","id":"14753073710734984378"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:09.815409208Z","hostname":"oxz_switch","pid":4920,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:11.296003131Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:11.300874871Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:11.303510639Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:12.205662791Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:12.210915397Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:12.214160263Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.515321988Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.519554589Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.522137366Z","hostname":"oxz_switch","pid":4920} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.52938839Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 649, tv_nsec: 11287517 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.529937058Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:13.561474551Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:14.642482914Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:14.648684143Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:14.652484407Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:15.306139739Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:15.311473245Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:15.314426768Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:16.716378031Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:16.720716844Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:16.723217379Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.315643091Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.320655774Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.323612676Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.842729425Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.848188213Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:17.851128206Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:18.532962322Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:19.156996968Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:19.160872103Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:19.163689064Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:19.817815183Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.283658196Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.288173851Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.290780038Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.878843468Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.883730878Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:20.886386016Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.195721708Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.200630149Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.202939512Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.947298087Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.952044656Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:22.954601723Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:23.557755188Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:23.56203463Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:23.564672608Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:24.543290818Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:24.547944375Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:24.550984879Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.215976503Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.000975284Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.003599652Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.639165304Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.648890944Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:25.653930716Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:26.568823691Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:26.574446087Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:26.577115653Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:27.990158521Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:27.995252161Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:27.99811933Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:28.322024889Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:28.349209439Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:28:28.350104862Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"20.036634726s","count":"0"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:28.66067097Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 664, tv_nsec: 355471952 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:28.661249723Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:28.693568817Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:29.465255837Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:29.477692725Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:29.482714121Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:29.612154436Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:30.438935415Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:30.445936393Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:30.449457987Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:31.723786699Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:31.730457921Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:31.734117141Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:33.074334235Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:33.080143899Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:33.083469338Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:34.506248086Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:34.511449094Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:34.513967254Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:35.639511019Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:35.64434642Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:35.647375057Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:36.891157143Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:36.8956866Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:36.898509666Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:38.046003629Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:38.051407599Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:38.054573977Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:38.35276221Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:39.035197467Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:39.041336405Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:39.044484975Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:39.642579491Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.287194301Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.292611213Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.295620484Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.960930772Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.96602642Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:40.968719339Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:42.262596662Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:42.267621734Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:42.270800088Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:43.087743203Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:43.093532691Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:43.096886813Z","hostname":"oxz_switch","pid":4920} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:43.737025638Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:28:43.737974067Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"35.38209328s","count":"1"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.016735753Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.021915815Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.024709549Z","hostname":"oxz_switch","pid":4920} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.349300497Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 680, tv_nsec: 893027 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.349918917Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:44.383140814Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:45.531046011Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:45.536557593Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:45.540138722Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:49.800988251Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:49.821682109Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:49.864744807Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:49.909189066Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:49.911694762Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.036175828Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.04326697Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.047609973Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.849136779Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.85687514Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:51.861286043Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.01421974Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.021344503Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.02605659Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.750981287Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.759241175Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:53.764172156Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:54.614982991Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:54.622313216Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:54.626224803Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:55.441190921Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:55.448214492Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:55.45224501Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:56.432606146Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:56.441092828Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:56.445185627Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:57.325775789Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:57.33140369Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:57.334278751Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:58.227977162Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:58.236343062Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:58.241540107Z","hostname":"oxz_switch","pid":4920} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.390324381Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:28:59.394638343Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"51.033566481s","count":"2"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.563032253Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.568663814Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.571644947Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.915493777Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:28:59.915920672Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.335000091Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 695, tv_nsec: 982790165 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.335526308Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.371110583Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.835453964Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.840337874Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:00.843098224Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:01.847404973Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:01.858256889Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:01.862802695Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:03.181441811Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:03.197835996Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:03.205563097Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:04.216031746Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:04.222490278Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:04.225934068Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:05.606434013Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:05.614298296Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:05.618367614Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:06.439698035Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:06.449157241Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:06.45396672Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:07.059172335Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:07.073132436Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:07.078673856Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:07.984809575Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:08.046797135Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:08.211202357Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:09.677049789Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:09.683916398Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:09.688204659Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:10.35784923Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:10.359592065Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:10.371328494Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:10.377944499Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:10.381988557Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:11.839113712Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:11.845727917Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:11.849488461Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:13.352431646Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:13.357352367Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:13.360175668Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:14.306099568Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:14.312219147Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:14.315850768Z","hostname":"oxz_switch","pid":4920} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:15.372335908Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:29:15.373770188Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"67.01280333s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:15.58681745Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:15.595539515Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:15.73301841Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:17.196603989Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:17.203115372Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:17.208183995Z","hostname":"oxz_switch","pid":4920} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.353724494Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 714, tv_nsec: 1808422 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.35414664Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.522910725Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.594893549Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.599971042Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:18.617196879Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:19.324537212Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:19.330485197Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:19.334104969Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:20.361915236Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:20.36214807Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:20.376682369Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:20.384202147Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:20.389157358Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:21.836267879Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:21.842162244Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:21.845627534Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:22.7689261Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:22.781843246Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:22.785549169Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:23.624351471Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:23.630970536Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:23.634516267Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:24.850163463Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:24.856795108Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:24.860631193Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:25.955836079Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:26.000098075Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:26.003548544Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.297358644Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.304401735Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.308636186Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.841903047Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.84899826Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:27.852620301Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:28.869205407Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:28.878213537Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:28.882323906Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:29.65024371Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:29.658729162Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:29.662175291Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:30.366638742Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:30.368944755Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:30.842392008Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:30.854566503Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:30.859893109Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:31.835996973Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:31.84691657Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:31.851813331Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:32.44609107Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:32.455660737Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:32.459446751Z","hostname":"oxz_switch","pid":4920} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:33.523873775Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:29:33.525226175Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"85.163940656s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:33.845451665Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:33.854614087Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:33.85976561Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:34.404111862Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:34.41023389Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:34.413459655Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:35.777486454Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:35.783205737Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:35.78621746Z","hostname":"oxz_switch","pid":4920} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.084505855Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 731, tv_nsec: 732287993 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.085417768Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.114808761Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.117362097Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", [fd00:1122:3344:101::c]:12221, Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })) })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2025-12-24T00:29:36.117934695Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/64b40cb/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"87.756616617s","count":"5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.526921242Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.532907028Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:36.536212965Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:37.668739657Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:37.674528681Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:37.678012431Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:38.877785999Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:38.885125144Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:38.888736946Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:40.273501392Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:40.279550889Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:40.282549052Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:40.370250372Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:40.370652258Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:41.765960126Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:41.774078542Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:41.778164121Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:42.463009751Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:42.469813618Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:42.473849096Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.105949588Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.112934048Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.117025247Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.76864797Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.775081802Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:43.778748915Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:44.340661628Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:44.345809952Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:44.349223601Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.168655475Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.174095213Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.177190088Z","hostname":"oxz_switch","pid":4920} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.190431928Z","hostname":"oxz_switch","pid":4920,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7ba64d96-4100-43cc-a77d-a592f61feb5c.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"7cb38847-4d2d-4ab1-aba0-5a654b70af69.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f3a11616-ac6b-4927-8b27-2017a1af3767.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::b)) }], valid_until: Instant { tv_sec: 740, tv_nsec: 838121686 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::c]:12221","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.190952225Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.221736708Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::c]:12221/metrics/producers","method":"POST"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.294206109Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Ok(Response { url: \"http://[fd00:1122:3344:101::c]:12221/metrics/producers\", status: 201, headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b4805382-8993-4096-ab21-32503389c3b2\", \"content-length\": \"41\", \"date\": \"Wed, 24 Dec 2025 00:29:45 GMT\"} })"} {"msg":"successfully registered with Nexus, pausing until time to renew lease","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:45.296669624Z","hostname":"oxz_switch","pid":4920,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","wait_period":"150s","lease_duration":"600s"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:46.59789907Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:46.603570742Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:46.606368812Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.270273961Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.276360229Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.279661456Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.983134423Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.989249001Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:47.992565389Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:48.771166956Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:48.777644999Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:48.781240271Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:49.296418142Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:49.302445839Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:49.305581184Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:50.302606869Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:50.308879389Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:50.31177188Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:50.37367473Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:50.374083566Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:51.564608681Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:51.57078091Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:51.574025196Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:52.831700026Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:52.838039706Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:52.841546357Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:53.957174146Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:53.962706825Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:53.965580747Z","hostname":"oxz_switch","pid":4920} {"msg":"accepted connection","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:29:54.905000923Z","hostname":"oxz_switch","pid":4920,"local_addr":"[fd00:1122:3344:101::2]:33932","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:1025","unit":"producer-server","unit":"oximeter","remote_addr":"[fd00:1122:3344:101::d]:60350"} {"msg":"request completed","v":0,"name":"tfportd","level":30,"time":"2025-12-24T00:29:54.912739585Z","hostname":"oxz_switch","pid":4920,"uri":"/fb29dcbd-b687-49e9-93de-abfd3492fd20","method":"GET","req_id":"7b0d7785-5828-4509-9631-fc388387ca00","remote_addr":"[fd00:1122:3344:101::d]:60350","local_addr":"[fd00:1122:3344:101::2]:33932","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.4/src/server.rs:867","unit":"producer-server","unit":"oximeter","latency_us":6525,"response_code":200} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.139414892Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.144753859Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.148360951Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.711165537Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.716684217Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:55.719989534Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:56.940958836Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:56.946911432Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:56.950195469Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:57.818691567Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:57.824660332Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:57.827493693Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:58.834176537Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:58.840084782Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:58.843005254Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:59.850277187Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:59.856292693Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:29:59.859827684Z","hostname":"oxz_switch","pid":4920} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:00.377666965Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10274436776871185655"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:00.378088781Z","hostname":"oxz_switch","pid":4920,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"14753073710734984378"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:00.723742327Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:00.731083183Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:00.735093901Z","hostname":"oxz_switch","pid":4920} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:01.463448675Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportrear0_0 addrs [fe80::4844:10ff:fe11:93e5]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:01.47280584Z","hostname":"oxz_switch","pid":4920} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2025-12-24T00:30:01.477159272Z","hostname":"oxz_switch","pid":4920}