[ May 16 04:27:20 Enabled. ] [ May 16 04:27:20 Rereading configuration. ] [ May 16 04:27:21 Rereading configuration. ] [ May 16 04:27:32 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ May 16 04:27:32 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.626316104Z","hostname":"oxz_switch","pid":5038,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-187aee7d","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.628194661Z","hostname":"oxz_switch","pid":5038} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.628393275Z","hostname":"oxz_switch","pid":5038} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.628839926Z","hostname":"oxz_switch","pid":5038,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.630896528Z","hostname":"oxz_switch","pid":5038,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n cdcd6d78-8882-4361-bf79-11e2afc3f63f,\n ),\n sled_id: Some(\n dca1fa3b-4844-4709-9635-47575ee8b3ec,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.637605534Z","hostname":"oxz_switch","pid":5038} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.637635875Z","hostname":"oxz_switch","pid":5038,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:32.640022634Z","hostname":"oxz_switch","pid":5038,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::70c0:71ff:fec4:c764->72:c0:71:c4:c7:64"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:32.643990802Z","hostname":"oxz_switch","pid":5038} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:32.651412186Z","hostname":"oxz_switch","pid":5038} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:27:32.657698012Z","hostname":"oxz_switch","pid":5038} [ May 16 04:27:33 Rereading configuration. ] [ May 16 04:27:33 Executing refresh method (:kill). ] [ May 16 04:27:33 Stopping because all processes in service exited. ] [ May 16 04:27:33 Executing stop method (:kill). ] [ May 16 04:27:33 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ May 16 04:27:33 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.391657264Z","hostname":"oxz_switch","pid":5445,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-187aee7d","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.39271222Z","hostname":"oxz_switch","pid":5445} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.393091719Z","hostname":"oxz_switch","pid":5445} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.393205512Z","hostname":"oxz_switch","pid":5445,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.39513785Z","hostname":"oxz_switch","pid":5445,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.395256543Z","hostname":"oxz_switch","pid":5445,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: cdcd6d78-8882-4361-bf79-11e2afc3f63f, sled_id: dca1fa3b-4844-4709-9635-47575ee8b3ec, model: \"i86pc\", revision: 0, serial: \"sock\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, fuse: Some(FuseData { chip_rev: ChipRevision { device_id: 272, rev: \"B1\", rev_num: 2 }, disabled: DisabledFeatures { bsync: false, mau: [0, 0, 0, 0], pgen: false, pipes: 0, ports: 0, resub: false, speeds: 0, tm_mem: 0 }, frequency: FrequencySettings { bps: 3, bps_ext: 0, cpu_speed_dis: 0, disabled: false, pcie_dis: 0, pps: 3, pps_ext: 0 }, manufacturing: ManufacturingData { core_repair: false, die_rotation: false, pmro_and_skew: 0, silent_spin: 0, soft_pipe_dis: 0, tile_repair: false, voltage_scaling: 0, wf_core_repair: false }, part: PartInfo { part_num: 4660, pkg_id: 1, version: 2 } }), lot: None, lotnum: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: b784867f-ca84-4a35-a426-9a3def137f22, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.395293664Z","hostname":"oxz_switch","pid":5445,"unit":"oximeter","producer_id":"a6e8113c-0a42-4e5d-8d6a-cc881e383af1"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n cdcd6d78-8882-4361-bf79-11e2afc3f63f,\n ),\n sled_id: Some(\n dca1fa3b-4844-4709-9635-47575ee8b3ec,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"sock\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.396244137Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.403803095Z","hostname":"oxz_switch","pid":5445} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.405902687Z","hostname":"oxz_switch","pid":5445,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::70c0:71ff:fec4:c764->72:c0:71:c4:c7:64"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.406146473Z","hostname":"oxz_switch","pid":5445,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::388c:ff:febc:3b43->3a:8c:00:bc:3b:43"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.407394414Z","hostname":"oxz_switch","pid":5445,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.408633224Z","hostname":"oxz_switch","pid":5445,"local_addr":"[fd00:1122:3344:101::2]:47695","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.408674045Z","hostname":"oxz_switch","pid":5445,"local_addr":"[fd00:1122:3344:101::2]:47695","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.4088474Z","hostname":"oxz_switch","pid":5445,"local_addr":"[fd00:1122:3344:101::2]:47695","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:47695","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.40887142Z","hostname":"oxz_switch","pid":5445,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.408878981Z","hostname":"oxz_switch","pid":5445,"unit":"producer-server","unit":"oximeter"} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.409062725Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.409420384Z","hostname":"oxz_switch","pid":5445,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.409471935Z","hostname":"oxz_switch","pid":5445,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:47695","producer_id":"a6e8113c-0a42-4e5d-8d6a-cc881e383af1"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.409484685Z","hostname":"oxz_switch","pid":5445,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.409490486Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.410154102Z","hostname":"oxz_switch","pid":5445} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.410521731Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.411971027Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-05-16T04:27:16.454910070Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.412000218Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","id":"11490380268656351138"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:33.412007888Z","hostname":"oxz_switch","pid":5445,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:27:33.413343911Z","hostname":"oxz_switch","pid":5445} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2026-05-16T04:27:33.990336895Z","hostname":"oxz_switch","pid":5445,"interface":"tfportqsfp0_0","entry":"10.151.2.174->f6:e6:cf:64:d2:b1"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.701862282Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.706361233Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.709549802Z","hostname":"oxz_switch","pid":5445} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.709796838Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.710843304Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-05-16T04:27:16.454777327Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.710881475Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","id":"3444092742692835361"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:34.710892745Z","hostname":"oxz_switch","pid":5445,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:36.174422251Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:36.180544393Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:36.184111981Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:37.465750991Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:37.470452278Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:37.473028662Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.412440434Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 669, tv_nsec: 460334900 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.413070959Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.429185649Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.962956502Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.967976506Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:38.970948519Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:39.918487523Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:39.923171929Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:39.926060401Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:41.05267986Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:41.058754641Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:41.061602791Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:42.505755258Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:42.510583127Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:42.513731535Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:43.415595417Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:43.455241209Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:43.459587887Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:43.46255745Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:44.396863196Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:44.401834669Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:44.404978667Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:44.711716426Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:45.797226167Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:45.80259562Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:45.805601805Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:46.720346076Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:46.725063223Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:46.728723863Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:47.356534486Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:47.361066808Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:47.364433381Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:49.209787947Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:49.214492204Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:49.217819316Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:50.055966236Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:50.061208566Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:50.064093318Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:51.220710066Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:51.225828379Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:51.228478148Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:52.355729873Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:52.360809093Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:52.36413351Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.422396377Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.435677336Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:27:53.436463272Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"20.022441325s","count":"0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.529356713Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.534085617Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.537210345Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.709460152Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 684, tv_nsec: 752087653 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.710152812Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:53.717908907Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:54.721072745Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.007461559Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.012273016Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.015900884Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.595793143Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.600930785Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:55.604609655Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:56.875540583Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:56.880244421Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:56.88346853Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.173926173Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.179273387Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.182343198Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.76558981Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.770655543Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:58.773696622Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:59.549042035Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:59.553681759Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:27:59.556590638Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:00.644973617Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:00.649763932Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:00.652724293Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:01.764695054Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:01.768750067Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:01.771810147Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.115878768Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.12004757Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.122889153Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.450267899Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.81324509Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.818022296Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:03.820947863Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:04.750299054Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:05.22766713Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:05.232076874Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:05.235583263Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:06.229300603Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:06.23339019Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:06.236535199Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:07.074919848Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:07.079841654Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:07.08262978Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.008951772Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.013977052Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.017149064Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.759802015Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:28:08.760776426Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"35.305362919s","count":"1"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.797542947Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.802139811Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:08.805305376Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:09.294148084Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 700, tv_nsec: 294758391 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:09.294832168Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:09.301292224Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:10.20283888Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:10.207988434Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:10.211777086Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:11.658456709Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:11.663591555Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:11.666344156Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.079013256Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.084783411Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.088441545Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.479072544Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.93747632Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.942192957Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:13.945743962Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:14.778712665Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:14.925871147Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:14.930372895Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:14.933698732Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:15.538573157Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:15.543526612Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:15.546806703Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:16.057345046Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:16.062788375Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:16.06628643Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:17.079517828Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:17.084478297Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:17.087406831Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:18.197908016Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:18.202693332Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:18.206518394Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:19.631024885Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:19.635946244Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:19.638704962Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:20.436435319Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:20.440659749Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:20.44394569Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:21.537162725Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:21.541643159Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:21.544815768Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:22.709591404Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:22.714601177Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:22.717247955Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:23.508616776Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:24.119675796Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:24.125153124Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:24.128224408Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:24.343798496Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:28:24.345367852Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"50.847951086s","count":"2"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:24.807748136Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.060453956Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 716, tv_nsec: 18523276 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.066580626Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.072966825Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.250412004Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.25539064Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.258947398Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.922466865Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.927071744Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:25.930493219Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:26.478060723Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:26.482434014Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:26.485806299Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:27.131742459Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:27.137043679Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:27.139925133Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:28.405195251Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:28.409786513Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:28.413161Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:29.714733472Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:29.7191839Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:29.722453371Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:31.159478564Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:31.163695918Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:31.16616912Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:32.536820799Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:32.541639337Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:32.545089247Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:33.538484023Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:33.593547081Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:33.60310559Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:33.6059497Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:34.83644137Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:34.904119103Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:34.910117234Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:34.913288458Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:36.278205177Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:36.282874018Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:36.285553708Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:37.465412415Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:37.47035071Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:37.472880922Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:38.844732642Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:38.849289999Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:38.85266503Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.115363129Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:28:40.116245708Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"66.576340935s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.147087906Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.15221911Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.155193125Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.939106294Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.943388645Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:40.946399527Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:41.287238302Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 732, tv_nsec: 201585199 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:41.28806345Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:41.296131315Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:42.171799968Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:42.176209076Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:42.179136877Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:43.567162302Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:43.641448572Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:43.646730476Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:43.650124429Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:44.666405155Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:44.671895155Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:44.675268368Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:44.865497392Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:45.661778231Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:45.666646696Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:45.670013864Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:46.345345178Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:46.351106923Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:46.353887081Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.087044137Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.091431576Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.094078143Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.701609122Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.706373057Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:47.709486988Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:48.450325632Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:48.455072154Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:48.458584408Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:49.329460633Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:49.333856473Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:49.337328178Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:50.765453664Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:50.771593916Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:50.774798113Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:51.607056182Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:51.611169658Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:51.614596118Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.067745599Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.071760668Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.074282705Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.596550866Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.914150176Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.919088171Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:53.922513446Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:54.711414635Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:54.716163194Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:54.719407458Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:54.894326398Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:55.269289295Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:55.27404937Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:55.277897753Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:56.337499074Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:28:56.33827264Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"82.754622029s","count":"4"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:56.472159827Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:56.476976817Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:56.479722729Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:57.746800875Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:57.751293149Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:57.754405046Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:58.81642931Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:58.822058479Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:58.825172002Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:28:59.996243297Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:00.000383499Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:00.003435912Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:00.820796747Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:00.825967345Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:00.828857312Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.296987099Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 752, tv_nsec: 157431205 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.297696874Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.305882674Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.900479715Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.905298253Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:01.908400848Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:02.670144121Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:02.675098902Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:02.678574436Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:03.425891785Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:03.43072308Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:03.433690033Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:03.62561333Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:04.387096506Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:04.391312742Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:04.394200757Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:04.923557044Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:05.476387436Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:05.480544456Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:05.484055639Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:06.939915251Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:06.944894257Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:06.948537688Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:08.000242721Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:08.004760758Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:08.007622194Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.13921289Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.14394107Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.146998354Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.732983351Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.737059309Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:09.739584424Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:11.174659585Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:11.180351873Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:11.183368029Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:12.528900391Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:12.533932334Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:12.537105104Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:13.654380958Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:13.690078128Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:13.694234074Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:13.698097449Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:14.291212294Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:14.296477906Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:14.299588435Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:14.952989605Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:15.546428584Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:15.550772137Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:15.55412765Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:16.347983487Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:29:16.348760822Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"102.711187833s","count":"5"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:16.837978011Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:16.842762847Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:16.845956307Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:18.082803005Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:18.086863536Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:18.089389466Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:19.067506455Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:19.073109479Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:19.076645183Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:20.582094365Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:20.586868825Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:20.589775549Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:22.024002244Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:22.028857826Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:22.03189887Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:23.486092058Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:23.490709319Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:23.49391263Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:23.683615448Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:24.40392098Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:24.408465369Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:24.412150937Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:24.981403847Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:25.394986264Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:25.399620516Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:25.402829587Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:26.099778922Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:26.105531393Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:26.108669199Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.225821148Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.23154456Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.234834162Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.891146983Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.896136839Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:27.899111055Z","hostname":"oxz_switch","pid":5445} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.25671523Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 779, tv_nsec: 44478668 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.257510425Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.265248042Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.507071369Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.51179379Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:28.515004885Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:29.866126681Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:29.871109228Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:29.87432071Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:30.881198093Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:30.885946596Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:30.889113758Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:31.521012514Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:31.525421465Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:31.52846832Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:32.291563142Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:32.295490646Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:32.29829094Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:33.494181237Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:33.499589654Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:33.503110997Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:33.712177068Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:34.61465298Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:34.619183631Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:34.622885431Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.010145125Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.303481469Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.308307512Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.312021462Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.857166134Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.862704804Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:35.865603427Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:37.379025015Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:37.384355319Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:37.387255508Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:38.598648253Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:38.603889893Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:38.60687729Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:39.610648093Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:39.615641082Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:39.619169261Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:40.664101737Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:40.668474982Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:40.671733422Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:41.351277403Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:41.355878676Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:41.359276386Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:42.580794601Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:42.585152444Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:42.588773348Z","hostname":"oxz_switch","pid":5445} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:43.307639689Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-05-16T04:29:43.308735182Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"129.598442731s","count":"6"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:43.408804217Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:43.414063776Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:43.417145263Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:43.741122024Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:44.885567715Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:44.890340973Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:44.894226488Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:45.038330676Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:45.881343918Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:45.885535279Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:45.888438125Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:47.084446845Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:47.090079681Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:47.092942101Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.080453547Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.084829139Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.087279836Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.988430071Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.993433228Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:48.99657636Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:50.214182238Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:50.218959527Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:50.222608419Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:51.482370658Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:51.486509233Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:51.489468848Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:52.43209962Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:52.436961976Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:52.440090926Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:53.476125062Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:53.480391382Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:53.483676259Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:53.770108015Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"11490380268656351138"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.395876133Z","hostname":"oxz_switch","pid":5445,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"957682b6-3ca6-4e84-b11a-7dd8a8366649.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"d1f6092c-34a6-4a6a-a2fe-73152fd5c29e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"f9044a8b-5917-4c1f-9fa1-299a4ed626fd.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 805, tv_nsec: 112988373 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.396620743Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.403400793Z","hostname":"oxz_switch","pid":5445,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.842678405Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.847791134Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:54.851329249Z","hostname":"oxz_switch","pid":5445} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:55.066188578Z","hostname":"oxz_switch","pid":5445,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"3444092742692835361"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.156447799Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.162167592Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.16539463Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.893641429Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.897965503Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:56.901285878Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:58.242477096Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:58.247591253Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:58.250412632Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:59.226032183Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:59.230404975Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:29:59.233701379Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:00.74730748Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:00.752332792Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:00.75562578Z","hostname":"oxz_switch","pid":5445} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:01.623377234Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportrear0_0 addrs [fe80::70c0:71ff:fec4:c764]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:01.632321869Z","hostname":"oxz_switch","pid":5445} {"msg":"sync tfportqsfp0_0 addrs [10.151.2.174]","v":0,"name":"tfportd","level":20,"time":"2026-05-16T04:30:01.63723401Z","hostname":"oxz_switch","pid":5445}