Max level shown:
12025-06-04T01:30:01.007ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 1203, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3."), ), original: Query { name: Name("oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3."), query_type: A, query_class: IN, }, }, original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 97, 51, 97, 51, 98, 101, 50, 101, 45, 102, 98, 48, 100, 45, 52, 49, 100, 48, 45, 57, 98, 98, 55, 45, 50, 52, 99, 101, 56, 53, 55, 102, 57, 101, 98, 51, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3." peer_addr = [fd00:1122:3344:101::7]:47250 req_id = 65828054-5b0f-4fb7-b2ed-32f5a3e666bf
22025-06-04T01:30:01.007ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 1203, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3."), ), original: Query { name: Name("oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3."), query_type: A, query_class: IN, }, }, original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 97, 51, 97, 51, 98, 101, 50, 101, 45, 102, 98, 48, 100, 45, 52, 49, 100, 48, 45, 57, 98, 98, 55, 45, 50, 52, 99, 101, 56, 53, 55, 102, 57, 101, 98, 51, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_a3a3be2e-fb0d-41d0-9bb7-24ce857f9eb3." peer_addr = [fd00:1122:3344:101::7]:58345 req_id = e81ee840-762e-44ae-9d5e-5097fbf76923
32025-06-04T01:30:01.026ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 23187, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d."), ), original: Query { name: Name("oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d."), query_type: A, query_class: IN, }, }, original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 53, 52, 98, 48, 50, 97, 51, 56, 45, 97, 99, 54, 98, 45, 52, 56, 54, 54, 45, 98, 102, 101, 54, 45, 99, 99, 51, 56, 51, 98, 51, 54, 56, 55, 50, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d." peer_addr = [fd00:1122:3344:101::6]:36667 req_id = a5702edc-7e34-4baa-8d45-938a0acdd6de
42025-06-04T01:30:01.027ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 23187, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d."), ), original: Query { name: Name("oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d."), query_type: A, query_class: IN, }, }, original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 53, 52, 98, 48, 50, 97, 51, 56, 45, 97, 99, 54, 98, 45, 52, 56, 54, 54, 45, 98, 102, 101, 54, 45, 99, 99, 51, 56, 51, 98, 51, 54, 56, 55, 50, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_54b02a38-ac6b-4866-bfe6-cc383b36872d." peer_addr = [fd00:1122:3344:101::6]:54801 req_id = b3094248-7dac-47b5-af47-1af09063061d
52025-06-04T01:30:09.332ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 64811, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:49373 req_id = 6b23cef7-bc1d-4f96-80ab-09c20556865e
62025-06-04T01:30:19.333ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 54444, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:54044 req_id = 57b248da-023f-4330-a8bd-19859b9f5811
72025-06-04T01:30:28.024ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::b]:53643
82025-06-04T01:30:28.066ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = b3a9a56a-b5d5-4579-833b-dc00428c14ac
92025-06-04T01:30:28.066ZINFOdns-server (store): updated generation new_generation = 1 req_id = b3a9a56a-b5d5-4579-833b-dc00428c14ac
102025-06-04T01:30:28.066ZINFOdns-server (http): request completed latency_us = 175 local_addr = [fd00:1122:3344:1::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::b]:53643 req_id = b3a9a56a-b5d5-4579-833b-dc00428c14ac response_code = 204 uri = /config
112025-06-04T01:30:29.334ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41299, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:52957 req_id = ded97620-ea77-42ba-8b6a-ffc5060a0fea
122025-06-04T01:30:29.460ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::c]:53246
132025-06-04T01:30:29.570ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 94282092-eb24-4f98-a5d9-c77b9c91bc99
142025-06-04T01:30:29.570ZINFOdns-server (store): updated generation new_generation = 1 req_id = 94282092-eb24-4f98-a5d9-c77b9c91bc99
152025-06-04T01:30:29.570ZINFOdns-server (http): request completed latency_us = 176 local_addr = [fd00:1122:3344:1::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::c]:53246 req_id = 94282092-eb24-4f98-a5d9-c77b9c91bc99 response_code = 204 uri = /config
162025-06-04T01:30:29.727ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::a]:57504
172025-06-04T01:30:29.778ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 3fe195d3-4632-4ee4-9aec-354f5fe4b767
182025-06-04T01:30:29.778ZINFOdns-server (store): updated generation new_generation = 1 req_id = 3fe195d3-4632-4ee4-9aec-354f5fe4b767
192025-06-04T01:30:29.778ZINFOdns-server (http): request completed latency_us = 168 local_addr = [fd00:1122:3344:1::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::a]:57504 req_id = 3fe195d3-4632-4ee4-9aec-354f5fe4b767 response_code = 204 uri = /config
202025-06-04T01:30:39.335ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 58474, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:61649 req_id = 811bf917-cac2-4335-901d-66bcbab5aa86
212025-06-04T01:30:49.337ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 38799, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:61452 req_id = e460a1ed-85ea-4376-b538-ad8b90c281d2
222025-06-04T01:30:59.339ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 57059, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:53708 req_id = c35d4439-1ad8-458c-8676-c9695870b9f5
232025-06-04T01:31:09.341ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62314, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:62818 req_id = e120f286-ee07-49e2-8458-b964abf75766
242025-06-04T01:31:19.342ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 36552, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, query: WireQuery { query: LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, dnssec_ok: false, max_payload: 1232, options: OPT { options: {}, }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:62931 req_id = 646521d1-a33a-450d-80cd-4a189b03ee83