[ Mar 7 01:43:40 Enabled. ] [ Mar 7 01:43:40 Rereading configuration. ] [ Mar 7 01:43:41 Rereading configuration. ] [ Mar 7 01:44:02 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Mar 7 01:44:02 Method "start" exited with status 0. ] No non-localhost IPv6 addresses found in SMF properties {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.227804678Z","hostname":"oxz_switch","pid":5411,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-f20f786e","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.231942073Z","hostname":"oxz_switch","pid":5411} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.232231793Z","hostname":"oxz_switch","pid":5411} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.232711848Z","hostname":"oxz_switch","pid":5411,"unit":"oximeter"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.234882819Z","hostname":"oxz_switch","pid":5411,"unit":"signal_handler"} No non-localhost IPv6 addresses found in SMF properties {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 793748c4-4beb-4252-87b7-4659feafcbd4,\n ),\n sled_id: Some(\n 875a97ff-610f-4476-8995-d9ae9d1e17d2,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.236096949Z","hostname":"oxz_switch","pid":5411} {"msg":"no non-localhost IPv6 listen address available, waiting for SMF update","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.236118109Z","hostname":"oxz_switch","pid":5411,"unit":"oximeter"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.240167582Z","hostname":"oxz_switch","pid":5411,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::b0b9:3eff:fe61:d0d->b2:b9:3e:61:0d:0d"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.242529819Z","hostname":"oxz_switch","pid":5411} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.247260803Z","hostname":"oxz_switch","pid":5411} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:02.252107011Z","hostname":"oxz_switch","pid":5411} [ Mar 7 01:44:02 Rereading configuration. ] [ Mar 7 01:44:02 Executing refresh method (:kill). ] [ Mar 7 01:44:02 Stopping because all processes in service exited. ] [ Mar 7 01:44:02 Executing stop method (:kill). ] [ Mar 7 01:44:02 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dendrite/bin/tfportd --sync-only &"). ] [ Mar 7 01:44:02 Method "start" exited with status 0. ] {"msg":"connecting to dpd","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.796614289Z","hostname":"oxz_switch","pid":5478,"port":12224,"host":"[::1]"} {"msg":"connected to dpd running softnpu-f20f786e","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.797488338Z","hostname":"oxz_switch","pid":5478} {"msg":"running in sync-only mode","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.797806708Z","hostname":"oxz_switch","pid":5478} {"msg":"found suitable IPv6 address for oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.797898821Z","hostname":"oxz_switch","pid":5478,"unit":"oximeter","address":"fd00:1122:3344:101::2"} {"msg":"refreshing SMF config prior to waiting on signals","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.799757002Z","hostname":"oxz_switch","pid":5478,"unit":"signal_handler"} {"msg":"extracted oximeter configuration data","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.799838554Z","hostname":"oxz_switch","pid":5478,"unit":"oximeter","config":"OximeterConfig { listen_address: fd00:1122:3344:101::2, sled_identifiers: SledIdentifiers { rack_id: 793748c4-4beb-4252-87b7-4659feafcbd4, sled_id: 875a97ff-610f-4476-8995-d9ae9d1e17d2, model: \"i86pc\", revision: 0, serial: \"buskin\" }, switch_identifiers: SwitchIdentifiers { asic_backend: \"softnpu\", fab: None, lot: None, model: \"FAKE_SIM_SIDECAR\", revision: 0, serial: \"SimSidecar0\", sidecar_id: d40e0341-5ba2-4a24-baf0-23290e0910b6, slot: 0, wafer: None, wafer_loc: None } }"} {"msg":"created producer ID","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.799853675Z","hostname":"oxz_switch","pid":5478,"unit":"oximeter","producer_id":"2a63dc7a-7146-4c00-b4d2-bcde6db8ccb0"} {"msg":"refreshed config: Config {\n log_file: None,\n log_format: Json,\n listen_addresses: [\n [fd00:1122:3344:101::2]:12231,\n [::1]:12231,\n ],\n pkt_source: None,\n dpd_host: \"[::1]\",\n dpd_port: 12224,\n vlan_link: None,\n vlan_data: None,\n techport0_prefix: None,\n techport1_prefix: None,\n rack_id: Some(\n 793748c4-4beb-4252-87b7-4659feafcbd4,\n ),\n sled_id: Some(\n 875a97ff-610f-4476-8995-d9ae9d1e17d2,\n ),\n sled_model: Some(\n \"i86pc\",\n ),\n sled_revision: Some(\n 0,\n ),\n sled_serial: Some(\n \"buskin\",\n ),\n}","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.800946271Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.808565829Z","hostname":"oxz_switch","pid":5478} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.809544991Z","hostname":"oxz_switch","pid":5478,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::b0b9:3eff:fe61:d0d->b2:b9:3e:61:0d:0d"} {"msg":"added new NDP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.809767848Z","hostname":"oxz_switch","pid":5478,"interface":"tfportrear0_0","entry":"tfportrear0_0:fe80::b0cd:28ff:fe2b:f2ac->b2:cd:28:2b:f2:ac"} {"msg":"registered DTrace probes","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.811232776Z","hostname":"oxz_switch","pid":5478,"unit":"producer-server","unit":"oximeter"} {"msg":"registered endpoint","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.812924411Z","hostname":"oxz_switch","pid":5478,"local_addr":"[fd00:1122:3344:101::2]:33397","component":"dropshot","unit":"producer-server","unit":"oximeter","versions":"all","path":"/{producer_id}","method":"GET"} {"msg":"listening","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.813057966Z","hostname":"oxz_switch","pid":5478,"local_addr":"[fd00:1122:3344:101::2]:33397","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.7/src/server.rs:328","unit":"producer-server","unit":"oximeter"} {"msg":"successfully registered DTrace USDT probes","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.813275073Z","hostname":"oxz_switch","pid":5478,"local_addr":"[fd00:1122:3344:101::2]:33397","component":"dropshot","unit":"producer-server","unit":"oximeter"} {"msg":"Requested any available port, Dropshot server has been bound to [fd00:1122:3344:101::2]:33397","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.813378436Z","hostname":"oxz_switch","pid":5478,"unit":"producer-server","unit":"oximeter"} {"msg":"Nexus IP not provided, will use DNS to resolve it","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.813389877Z","hostname":"oxz_switch","pid":5478,"unit":"producer-server","unit":"oximeter"} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.813421867Z","hostname":"oxz_switch","pid":5478} {"msg":"new DNS resolver","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.813846091Z","hostname":"oxz_switch","pid":5478,"component":"internal-dns-resolver","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/internal-dns/resolver/src/resolver.rs:144","unit":"producer-server","unit":"oximeter","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting producer registration task","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.814828153Z","hostname":"oxz_switch","pid":5478,"unit":"producer-server","unit":"oximeter"} {"msg":"starting oximeter metric producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.814977738Z","hostname":"oxz_switch","pid":5478,"file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:279","unit":"producer-server","unit":"oximeter","interval":"30s","address":"[fd00:1122:3344:101::2]:33397","producer_id":"2a63dc7a-7146-4c00-b4d2-bcde6db8ccb0"} {"msg":"created oximeter producer server","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:02.814996309Z","hostname":"oxz_switch","pid":5478,"unit":"oximeter"} {"msg":"registering as oximeter producer with Nexus","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.815003209Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","registration_count":"0"} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.8150305Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.816046233Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-03-07T01:43:36.211416095Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportrear0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.816078374Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","id":"1092800514650368985"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:02.816107195Z","hostname":"oxz_switch","pid":5478,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportrear0_0"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:02.816526509Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"proto error: no records found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"1.518819ms","count":"0"} {"msg":"tfportqsfp0_0 has no addrs","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:02.816593351Z","hostname":"oxz_switch","pid":5478} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:03.099268665Z","hostname":"oxz_switch","pid":5478,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"3b725edd-3111-4340-9cbb-becbd792b090.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"67fe79d4-b7f4-4e94-a920-250e511ba133.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 1540, tv_nsec: 230071298 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:03.099969068Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:03.144375497Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"added new ARP entry","v":0,"name":"tfportd","level":30,"time":"2026-03-07T01:44:03.677606637Z","hostname":"oxz_switch","pid":5478,"interface":"tfportqsfp0_0","entry":"10.151.1.174->96:be:78:d7:ad:d6"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.142265088Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.148611385Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.162040343Z","hostname":"oxz_switch","pid":5478} {"msg":"adding new target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.162411885Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler","unit":"oximeter","name":"switch_port_control_data_link"} {"msg":"storing new creation time for tracked target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.163858463Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","creation_time":"2026-03-07T01:43:36.211277731Z","path":"KstatPath { module: \"link\", instance: 0, name: \"tfportqsfp0_0\" }"} {"msg":"inserted empty per-target sample queue","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.163881673Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","id":"10573277605096125527"} {"msg":"added new link to kstat sampler","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:04.163898144Z","hostname":"oxz_switch","pid":5478,"unit":"metrics-task-tracker","unit":"oximeter","link_kind":"switch-port-control","link_name":"tfportqsfp0_0"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:05.317639821Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:05.322761639Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:05.32557548Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:06.408113284Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:06.412538448Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:06.415509725Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:07.318160278Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:07.327905427Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:07.330506171Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:08.444826402Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:08.449284277Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:08.451742767Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.069754074Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.075735979Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.079819452Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.86610353Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.870734191Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:09.873442579Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:10.682577462Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:10.687990969Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:10.691078129Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:11.779784284Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:11.78489001Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:11.78762791Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:12.821532086Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1092800514650368985"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:12.888106699Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:12.892818382Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:12.895321334Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:13.543655259Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:13.549150038Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:13.552136015Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.166793612Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10573277605096125527"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.212557196Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.217848368Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.220550086Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.82559064Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.830083996Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:14.832904328Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:15.683603346Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:15.688068852Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:15.690727779Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:16.433911928Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:16.437889518Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:16.44039405Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:17.037653281Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:17.043180061Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:17.045929682Z","hostname":"oxz_switch","pid":5478} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.14005039Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:18.141184047Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"15.331965348s","count":"1"} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.434086829Z","hostname":"oxz_switch","pid":5478,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"3b725edd-3111-4340-9cbb-becbd792b090.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"67fe79d4-b7f4-4e94-a920-250e511ba133.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 1555, tv_nsec: 571566098 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.434687229Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.466602833Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.501878187Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.507577424Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:18.511165811Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:19.039830865Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:19.05389554Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:19.159454789Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:20.59266194Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:20.597990844Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:20.600765335Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:21.327348277Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:21.333705164Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:21.337153957Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:22.49723626Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:22.507369821Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:22.514336738Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:22.811126831Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1092800514650368985"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:23.281391962Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:23.288892745Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:23.29341725Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:24.158171593Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10573277605096125527"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:24.715839508Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:24.723026288Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:24.726417456Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.225980171Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.230965968Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.233829686Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.80889508Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.814048023Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:26.816921253Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:27.959195701Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:27.964630579Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:27.967918762Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:28.566845493Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:28.573527931Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:28.577624565Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:29.308958316Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:29.315315274Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:29.319094037Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.515215632Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.543464744Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1092800514650368985"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.544734965Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10573277605096125527"} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.550265116Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.570127544Z","hostname":"oxz_switch","pid":5478} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:34.636457619Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"31.830603386s","count":"2"} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:34.645896157Z","hostname":"oxz_switch","pid":5478} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.378692196Z","hostname":"oxz_switch","pid":5478,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"3b725edd-3111-4340-9cbb-becbd792b090.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"67fe79d4-b7f4-4e94-a920-250e511ba133.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 1572, tv_nsec: 518462742 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.579820121Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.655584614Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.680762026Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.68853999Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:35.69374814Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:36.655010235Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:36.662413957Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:36.666878103Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:37.920266581Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:37.925538682Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:37.930419721Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:38.588721911Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:38.595615135Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:38.600028649Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:39.778601995Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:39.784716044Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:39.788162577Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:41.300992648Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:41.307777098Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:41.311887738Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:42.299685862Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:42.305179802Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:42.308090827Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:43.547284456Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:43.553682955Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:43.55719957Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:44.379524641Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:44.384138091Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:44.386880641Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:44.548304518Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1092800514650368985"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:44.54867138Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10573277605096125527"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.345434468Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.350149652Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.353130169Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.918273971Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.922518009Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:45.925076493Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:47.330603927Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:47.335489936Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:47.338591118Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:48.160590921Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:48.16517938Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:48.167793696Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:49.251889542Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:49.257095052Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:49.259812641Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:50.062414711Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:50.068177409Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:50.071133695Z","hostname":"oxz_switch","pid":5478} {"msg":"client response","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:50.657919264Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","result":"Err(reqwest::Error { kind: Request, url: \"http://[fd00:1122:3344:101::a]:12221/metrics/producers\", source: TimedOut })"} {"msg":"failed to register with Nexus, will retry","v":0,"name":"tfportd","level":40,"time":"2026-03-07T01:44:50.660135126Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","file":"/home/build/.cargo/git/checkouts/omicron-e0115f488d1c1d58/8e30605/oximeter/producer/src/lib.rs:371","unit":"producer-server","unit":"oximeter","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"47.856453816s","count":"3"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.401666424Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.406812622Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.409685926Z","hostname":"oxz_switch","pid":5478} {"msg":"lookup_socket_v6 srv","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.724608702Z","hostname":"oxz_switch","pid":5478,"component":"internal-dns-resolver","unit":"producer-server","unit":"oximeter","response":"SrvLookup(Lookup { query: Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }, records: [Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"3b725edd-3111-4340-9cbb-becbd792b090.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"67fe79d4-b7f4-4e94-a920-250e511ba133.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: SRV(SRV { priority: 0, weight: 0, port: 12221, target: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\") }) }, Record { name_labels: Name(\"cfb08bc8-1733-4a6a-b6ce-e1345a48e55e.host.control-plane.oxide.internal.\"), dns_class: IN, ttl: 0, rdata: AAAA(AAAA(fd00:1122:3344:101::c)) }], valid_until: Instant { tv_sec: 1588, tv_nsec: 866879248 } })","dns_name":"_nexus._tcp.control-plane.oxide.internal"} {"msg":"will register with Nexus at [fd00:1122:3344:101::a]:12221","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.725412678Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter"} {"msg":"client request","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:51.75917577Z","hostname":"oxz_switch","pid":5478,"component":"producer-registration-task","unit":"producer-server","unit":"oximeter","body":"Some(Body)","uri":"http://[fd00:1122:3344:101::a]:12221/metrics/producers","method":"POST"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:52.810728804Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:52.816007356Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:52.819247192Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:53.528307808Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:53.534085967Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:53.537107896Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:54.192912445Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:54.198136856Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:54.201424773Z","hostname":"oxz_switch","pid":5478} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:54.549594884Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"1092800514650368985"} {"msg":"pulled samples from target","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:54.549996317Z","hostname":"oxz_switch","pid":5478,"component":"kstat-sampler-worker","unit":"oximeter","n_samples":6,"id":"10573277605096125527"} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:55.372694441Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:55.378216191Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:55.380975171Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:56.135428669Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:56.140868756Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:56.143974548Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:57.168146916Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:57.172817458Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:57.175512676Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:58.457047962Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:58.462677786Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:58.465767067Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:59.364386999Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:59.370352663Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:44:59.373442654Z","hostname":"oxz_switch","pid":5478} {"msg":"found simports [\n \"tfportrear0_0\",\n \"tfportqsfp0_0\",\n]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:45:00.499154587Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportrear0_0 addrs [fe80::b0b9:3eff:fe61:d0d]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:45:00.505273227Z","hostname":"oxz_switch","pid":5478} {"msg":"sync tfportqsfp0_0 addrs [10.151.1.174]","v":0,"name":"tfportd","level":20,"time":"2026-03-07T01:45:00.508758431Z","hostname":"oxz_switch","pid":5478}