Max level shown:
1 [ Jul 21 18:12:08 Enabled. ]
2 [ Jul 21 18:12:08 Rereading configuration. ]
3 [ Jul 21 18:12:08 Rereading configuration. ]
4 [ Jul 21 18:12:09 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ]
5 [ Jul 21 18:12:09 Method "start" exited with status 0. ]
6 Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
7 note: configured to log to "/dev/stdout"
82026-07-21T18:12:09.805ZINFOdns-server: config config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } dns_config = Config { bind_address: [fd00:1122:3344:1::1]:53 }
92026-07-21T18:12:09.805ZINFOdns-server (store): opening sled database path = /data/dns
102026-07-21T18:12:09.825ZINFOdns-server (store): pruning trees for generations newer than 0
112026-07-21T18:12:09.825ZINFOdns-server (store): pruning trees for generations older than 0 keep = 3
122026-07-21T18:12:09.825ZINFOdns-server (dns): DNS server bound to address local_address = [fd00:1122:3344:1::1]:53
132026-07-21T18:12:09.826ZINFOdns-server (http): listening local_addr = [fd00:1122:3344:1::1]:5353
142026-07-21T18:12:09.832ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 56103, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:34634 req_id = 7f4a8629-baf5-4a62-aa6f-1fabf31fd74e
152026-07-21T18:12:09.946ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60951, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:32710 req_id = 82219b70-8000-4062-a3ef-77768cb3ee43
162026-07-21T18:12:10.102ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12290, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:9738 req_id = 543c540f-258d-4219-b087-69ea11daa930
172026-07-21T18:12:10.118ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 54888, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:12908 req_id = c3b1e5e5-0c4b-4145-b0a8-8745acc4cf18
182026-07-21T18:12:10.283ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41824, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:29578 req_id = cae6c8d0-5ccf-4750-9f6a-e747e5460a37
192026-07-21T18:12:10.331ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 61033, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:19890 req_id = dc3b6c9e-73cc-40f0-9a33-f0d289bc6b48
202026-07-21T18:12:10.488ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 14628, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:18239 req_id = 55799ed1-4d58-4b04-9b94-27772820df7c
212026-07-21T18:12:10.775ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4463, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:45880 req_id = e9ae0557-dc55-489b-87a8-0034d1918013
222026-07-21T18:12:10.825ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 29202, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:30548 req_id = 38169c1a-17bf-4a67-b38c-4ffe9e522ef8
232026-07-21T18:12:11.192ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37742, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:1::2]:22379 req_id = ebaa00c5-e3f8-4d9f-90a2-b0128d458ce3
242026-07-21T18:12:11.220ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60683, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:42978 req_id = e8f07823-0c74-4eda-91c3-0e20a79ac6d0
252026-07-21T18:12:11.323ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:1::2]:57164
262026-07-21T18:12:11.324ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 7ea25e73-0421-4d8a-9fa6-a7045972d254
272026-07-21T18:12:11.324ZINFOdns-server (store): pruning trees for generations newer than 0
282026-07-21T18:12:11.346ZINFOdns-server (store): pruning trees for generations older than 1 keep = 3
292026-07-21T18:12:11.346ZINFOdns-server (store): updated generation new_generation = 1 req_id = 7ea25e73-0421-4d8a-9fa6-a7045972d254
302026-07-21T18:12:11.346ZINFOdns-server (http): request completed latency_us = 22515 local_addr = [fd00:1122:3344:1::1]:5353 method = PUT remote_addr = [fd00:1122:3344:1::2]:57164 req_id = 7ea25e73-0421-4d8a-9fa6-a7045972d254 response_code = 204 uri = /config
312026-07-21T18:12:24.164ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 35471, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:48063 req_id = fa2a53d4-f801-4538-aec8-0dd72f7a33d5
322026-07-21T18:12:24.169ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 686, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:62861 req_id = a24b110e-74c5-496e-913c-09cd02949a25
332026-07-21T18:12:24.173ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 1141, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:40265 req_id = 44884268-5f4a-40eb-a3be-4667dbe34ca4
342026-07-21T18:14:04.917ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 17527, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:63031 req_id = 2b03848b-eb01-4f80-a34e-8f8a3fefb987
352026-07-21T18:14:07.128ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 30711, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:59332 req_id = e02f18e5-dc80-46a1-8d8b-c88de3c0f6ed
362026-07-21T18:14:07.139ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 30711, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:48290 req_id = b21e3ac7-0d95-4e0a-ad83-7b77534e1770
372026-07-21T18:14:14.907ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62534, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:63699 req_id = d2f50979-b073-4b33-84b5-5c7e00aabfe6
382026-07-21T18:14:15.922ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44194, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:23735 req_id = 6b3e59af-d2be-44fb-b964-95b7c065d7e4
392026-07-21T18:14:15.922ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 34625, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:12577 req_id = e2f7dbea-5777-408b-9f29-9b24f5499615
402026-07-21T18:14:15.975ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49505, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:7005 req_id = ac51ddd0-1760-43f9-8017-50f97b94f5f6
412026-07-21T18:14:24.908ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 58216, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:50043 req_id = 3e2bb2e5-f91b-4d87-9b85-12b192335648
422026-07-21T18:14:28.949ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 13401, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:6785 req_id = b8c541c8-737f-42b2-b8ae-bf2e432ce33a
432026-07-21T18:14:29.181ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::c]:59322
442026-07-21T18:14:29.224ZINFOdns-server (http): request completed latency_us = 24773 local_addr = [fd00:1122:3344:1::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:59322 req_id = a97ec491-6f98-483b-a339-0cf2561c4c34 response_code = 200 uri = /config
452026-07-21T18:14:33.181ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::c]:41145
462026-07-21T18:14:33.188ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = f390875d-aa0a-4b73-b3f9-45f27edfb630
472026-07-21T18:14:33.188ZINFOdns-server (store): updated generation new_generation = 1 req_id = f390875d-aa0a-4b73-b3f9-45f27edfb630
482026-07-21T18:14:33.189ZINFOdns-server (http): request completed latency_us = 5388 local_addr = [fd00:1122:3344:1::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::c]:41145 req_id = f390875d-aa0a-4b73-b3f9-45f27edfb630 response_code = 204 uri = /config
492026-07-21T18:14:33.397ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49295, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:6618 req_id = 7133251b-5f87-4acd-b45d-8edecd4e15a4
502026-07-21T18:14:33.520ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::c]:35905
512026-07-21T18:14:33.520ZINFOdns-server (http): request completed latency_us = 182 local_addr = [fd00:1122:3344:1::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:35905 req_id = 6ff7c6bd-699d-4d76-b1fa-108a37493d6f response_code = 200 uri = /config
522026-07-21T18:14:33.868ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 42368, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:60425 req_id = 03802c79-0670-463c-8bb3-8a54b3754d9f
532026-07-21T18:14:33.988ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::c]:40976
542026-07-21T18:14:33.988ZINFOdns-server (http): request completed latency_us = 237 local_addr = [fd00:1122:3344:1::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:40976 req_id = 685dbfed-0cd1-4ebf-a4ff-2d1393fd2178 response_code = 200 uri = /config
552026-07-21T18:14:34.910ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41805, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:64618 req_id = dd298460-9cbd-4dd7-bbb7-889d7f280248
562026-07-21T18:14:36.391ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 42058, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:56194 req_id = c20fa26f-cee3-45d2-87b6-e5bb246e3dc8
572026-07-21T18:14:36.502ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12731, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:12218 req_id = c2091b5f-9a67-43f6-8e34-d9c2ea10e718
582026-07-21T18:14:36.523ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::a]:58811
592026-07-21T18:14:36.523ZINFOdns-server (http): request completed latency_us = 261 local_addr = [fd00:1122:3344:1::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:58811 req_id = bb8039a1-5aee-42cd-8b85-bfd67a697cc0 response_code = 200 uri = /config
602026-07-21T18:14:36.731ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:1::1]:5353 remote_addr = [fd00:1122:3344:101::b]:62105
612026-07-21T18:14:36.731ZINFOdns-server (http): request completed latency_us = 253 local_addr = [fd00:1122:3344:1::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::b]:62105 req_id = 78f1e25c-b550-4d63-92f0-b868628449aa response_code = 200 uri = /config
622026-07-21T18:14:44.911ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 24948, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:64568 req_id = 6507fbbe-1c44-47fa-a137-7bb07bb9d674
632026-07-21T18:14:54.912ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37631, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:59239 req_id = aa11749d-61f6-4dc3-9b72-941e13b4af3f
642026-07-21T18:15:00.174ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49781, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), ), original: Query { name: Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 98, 97, 56, 51, 55, 52, 45, 55, 101, 97, 50, 45, 52, 49, 57, 50, 45, 57, 56, 99, 97, 45, 56, 53, 48, 101, 56, 102, 53, 51, 100, 101, 49, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10." peer_addr = [fd00:1122:3344:101::7]:35446 req_id = 484a9a90-bf76-462a-acb7-14b84d4a204d
652026-07-21T18:15:00.174ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49781, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), ), original: Query { name: Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 98, 97, 56, 51, 55, 52, 45, 55, 101, 97, 50, 45, 52, 49, 57, 50, 45, 57, 56, 99, 97, 45, 56, 53, 48, 101, 56, 102, 53, 51, 100, 101, 49, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10." peer_addr = [fd00:1122:3344:101::7]:58749 req_id = d4221838-e389-4a16-852d-48d1827e4b03
662026-07-21T18:15:00.262ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52016, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), ), original: Query { name: Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 99, 55, 98, 56, 51, 51, 53, 57, 45, 48, 51, 55, 99, 45, 52, 49, 55, 100, 45, 56, 51, 102, 53, 45, 98, 55, 49, 101, 99, 53, 50, 48, 56, 54, 100, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd." peer_addr = [fd00:1122:3344:101::4]:35361 req_id = a6d0ce62-a4b4-4ca9-83c4-38a902e7986c
672026-07-21T18:15:00.263ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52016, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), ), original: Query { name: Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 99, 55, 98, 56, 51, 51, 53, 57, 45, 48, 51, 55, 99, 45, 52, 49, 55, 100, 45, 56, 51, 102, 53, 45, 98, 55, 49, 101, 99, 53, 50, 48, 56, 54, 100, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd." peer_addr = [fd00:1122:3344:101::4]:41524 req_id = c3519120-b95f-4289-9a2a-b6b91e4fd6e9
682026-07-21T18:15:00.291ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 7382, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), ), original: Query { name: Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 100, 51, 53, 102, 56, 97, 55, 45, 51, 100, 49, 98, 45, 52, 51, 52, 99, 45, 57, 102, 57, 51, 45, 99, 48, 54, 49, 55, 51, 54, 101, 100, 53, 102, 56, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8." peer_addr = [fd00:1122:3344:101::3]:33327 req_id = 36f1c048-9c6c-45de-9242-e75051734d78
692026-07-21T18:15:00.291ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 7382, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), ), original: Query { name: Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 100, 51, 53, 102, 56, 97, 55, 45, 51, 100, 49, 98, 45, 52, 51, 52, 99, 45, 57, 102, 57, 51, 45, 99, 48, 54, 49, 55, 51, 54, 101, 100, 53, 102, 56, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8." peer_addr = [fd00:1122:3344:101::3]:59823 req_id = c51213ca-3c15-4ddd-995e-cfe08d6b6fe1
702026-07-21T18:15:00.296ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40937, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), ), original: Query { name: Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 48, 100, 100, 51, 101, 49, 55, 45, 57, 50, 55, 48, 45, 52, 53, 57, 98, 45, 98, 98, 99, 55, 45, 49, 51, 54, 101, 55, 57, 102, 99, 54, 54, 56, 54, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686." peer_addr = [fd00:1122:3344:101::5]:56820 req_id = 36e65010-7f42-41b4-ab8f-2d64f56e1222
712026-07-21T18:15:00.297ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40937, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), ), original: Query { name: Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 48, 100, 100, 51, 101, 49, 55, 45, 57, 50, 55, 48, 45, 52, 53, 57, 98, 45, 98, 98, 99, 55, 45, 49, 51, 54, 101, 55, 57, 102, 99, 54, 54, 56, 54, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686." peer_addr = [fd00:1122:3344:101::5]:46827 req_id = 7dc0f609-22d1-4b84-ad95-0ac88632fda7
722026-07-21T18:15:00.313ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 64288, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5."), ), original: Query { name: Name("oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5."), query_type: A, query_class: IN, }, }, ], original: [ 44, 111, 120, 122, 95, 110, 116, 112, 95, 51, 100, 55, 97, 57, 53, 51, 101, 45, 102, 57, 98, 55, 45, 52, 48, 55, 99, 45, 97, 49, 48, 97, 45, 51, 55, 51, 50, 101, 55, 51, 53, 51, 101, 98, 53, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5." peer_addr = [fd00:1122:3344:101::1b]:62452 req_id = 72e6850e-9bd2-4be0-9c8b-91ee9f58d060
732026-07-21T18:15:00.461ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26419, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:49486 req_id = 7bcdba06-0811-42c7-b6e0-c880759750d2
742026-07-21T18:15:00.462ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26419, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:37717 req_id = 812ed79a-748c-416e-b9d9-f5a8b9202d91
752026-07-21T18:15:00.469ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18994, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), ), original: Query { name: Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 102, 53, 52, 57, 48, 51, 45, 52, 99, 57, 52, 45, 52, 102, 50, 53, 45, 98, 56, 50, 53, 45, 98, 51, 57, 100, 99, 101, 97, 100, 102, 49, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4." peer_addr = [fd00:1122:3344:101::6]:48257 req_id = 7080e954-508a-4b7a-a47a-7b9af81e244e
762026-07-21T18:15:00.470ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18994, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), ), original: Query { name: Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 102, 53, 52, 57, 48, 51, 45, 52, 99, 57, 52, 45, 52, 102, 50, 53, 45, 98, 56, 50, 53, 45, 98, 51, 57, 100, 99, 101, 97, 100, 102, 49, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4." peer_addr = [fd00:1122:3344:101::6]:57136 req_id = 1c8f8618-fef2-489e-a86e-1e177ac9ed51
772026-07-21T18:15:00.505ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44421, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:45226 req_id = f971cf37-4935-4ce9-bc04-b230c3f9d5a2
782026-07-21T18:15:00.505ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44421, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:43429 req_id = 10b25008-a4f3-487b-9211-b54703131ec6