Max level shown:
1 [ Jul 21 18:12:07 Enabled. ]
2 [ Jul 21 18:12:07 Rereading configuration. ]
3 [ Jul 21 18:12:08 Rereading configuration. ]
4 [ Jul 21 18:12:09 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ]
5 [ Jul 21 18:12:09 Method "start" exited with status 0. ]
6 Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
7 note: configured to log to "/dev/stdout"
82026-07-21T18:12:09.782ZINFOdns-server: config config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } dns_config = Config { bind_address: [fd00:1122:3344:2::1]:53 }
92026-07-21T18:12:09.783ZINFOdns-server (store): opening sled database path = /data/dns
102026-07-21T18:12:09.803ZINFOdns-server (store): pruning trees for generations newer than 0
112026-07-21T18:12:09.804ZINFOdns-server (store): pruning trees for generations older than 0 keep = 3
122026-07-21T18:12:09.804ZINFOdns-server (dns): DNS server bound to address local_address = [fd00:1122:3344:2::1]:53
132026-07-21T18:12:09.804ZINFOdns-server (http): listening local_addr = [fd00:1122:3344:2::1]:5353
142026-07-21T18:12:09.832ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 64922, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:41014 req_id = f440fe39-8667-4565-ae7f-9898f7be7798
152026-07-21T18:12:09.946ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18694, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:52433 req_id = ea02b19f-57e0-4c30-a9fa-7d13a37ad89b
162026-07-21T18:12:10.102ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 15044, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:38282 req_id = 8f8a9dee-f630-4546-bd90-17e1c29ba6a7
172026-07-21T18:12:10.118ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 47424, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:51527 req_id = 3db4ecca-38d8-437e-b30c-7db63fc1e035
182026-07-21T18:12:10.284ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51794, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:39397 req_id = 5e715a81-414c-45ea-8d64-63829e684907
192026-07-21T18:12:10.331ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44846, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:37998 req_id = b3b11718-0ff8-4356-a35b-792e205b0501
202026-07-21T18:12:10.488ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 45106, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:17799 req_id = 9007993c-75cd-4acc-beec-c05090c2a48a
212026-07-21T18:12:10.775ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 58584, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:59171 req_id = 049c6400-f8dc-47ab-9c9d-3a0a19fb3051
222026-07-21T18:12:10.825ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 6573, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:24944 req_id = 2dc06632-9a12-442a-8a06-cf0bb5cdd4d8
232026-07-21T18:12:11.192ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49958, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:2::2]:20515 req_id = 71b4025c-8bdd-4c1b-baf2-9482845eaf1d
242026-07-21T18:12:11.220ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 57076, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:63083 req_id = 5bb97336-ab31-4e1c-a0ca-129597313669
252026-07-21T18:12:11.347ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:2::2]:52727
262026-07-21T18:12:11.348ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 88ac3c1d-4317-4b90-aae7-c4682be09427
272026-07-21T18:12:11.348ZINFOdns-server (store): pruning trees for generations newer than 0
282026-07-21T18:12:11.369ZINFOdns-server (store): pruning trees for generations older than 1 keep = 3
292026-07-21T18:12:11.369ZINFOdns-server (store): updated generation new_generation = 1 req_id = 88ac3c1d-4317-4b90-aae7-c4682be09427
302026-07-21T18:12:11.369ZINFOdns-server (http): request completed latency_us = 21239 local_addr = [fd00:1122:3344:2::1]:5353 method = PUT remote_addr = [fd00:1122:3344:2::2]:52727 req_id = 88ac3c1d-4317-4b90-aae7-c4682be09427 response_code = 204 uri = /config
312026-07-21T18:12:24.164ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 35471, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:58573 req_id = d0ce3719-8660-4ef5-90f5-534780470ac2
322026-07-21T18:12:24.169ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 686, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:50857 req_id = 35137c01-0693-46d6-a1f7-55c45091ca44
332026-07-21T18:12:24.173ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 1141, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:65102 req_id = cc8e512c-3dbb-44ab-840c-de88f65d671f
342026-07-21T18:14:04.935ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 29436, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:55647 req_id = f6c17e03-6299-42fa-bd14-312516462b2c
352026-07-21T18:14:07.133ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 30711, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:61096 req_id = 1f994228-e565-46aa-be94-e36fa51da46f
362026-07-21T18:14:07.140ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 30711, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:47111 req_id = ea654593-6d2a-4ebb-b3ab-9ded09c776d6
372026-07-21T18:14:14.907ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37067, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:61041 req_id = 4271bafa-37a6-4c9f-b3a2-4de3daa4d8e8
382026-07-21T18:14:15.922ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51934, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:25053 req_id = a5ec12b7-8896-48c4-ab5a-a27afc59f983
392026-07-21T18:14:15.923ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 47359, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:6278 req_id = b344d691-d027-4763-884c-559ed3586fc4
402026-07-21T18:14:15.975ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26130, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:50297 req_id = 5e448e73-fcb4-4ad2-af37-adee8a4f2f92
412026-07-21T18:14:24.908ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 24291, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:54836 req_id = f8ae6957-d8cc-4716-a7bf-adac8ffdaa0a
422026-07-21T18:14:28.949ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 5796, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:16225 req_id = 598a6d5a-614c-466a-b5a4-6f50271fdde2
432026-07-21T18:14:29.243ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::c]:36711
442026-07-21T18:14:29.295ZINFOdns-server (http): request completed latency_us = 31792 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:36711 req_id = 06ab9345-b74b-4c78-95cb-42690818e377 response_code = 200 uri = /config
452026-07-21T18:14:33.182ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::c]:41634
462026-07-21T18:14:33.186ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = cb5f1cdb-1467-4399-b998-496d081dfb68
472026-07-21T18:14:33.186ZINFOdns-server (store): updated generation new_generation = 1 req_id = cb5f1cdb-1467-4399-b998-496d081dfb68
482026-07-21T18:14:33.186ZINFOdns-server (http): request completed latency_us = 2642 local_addr = [fd00:1122:3344:2::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::c]:41634 req_id = cb5f1cdb-1467-4399-b998-496d081dfb68 response_code = 204 uri = /config
492026-07-21T18:14:33.397ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 54943, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:33761 req_id = 47c4fc3a-3edd-4a1b-88e7-d868e097cb6d
502026-07-21T18:14:33.521ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::c]:60951
512026-07-21T18:14:33.521ZINFOdns-server (http): request completed latency_us = 231 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:60951 req_id = 49f0cee4-8ee8-4b12-bdc0-b04313c9f2c6 response_code = 200 uri = /config
522026-07-21T18:14:33.868ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 63252, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:40418 req_id = fe3c6409-6978-4bfb-8e76-3e19e637fc30
532026-07-21T18:14:33.989ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::c]:34964
542026-07-21T18:14:33.989ZINFOdns-server (http): request completed latency_us = 191 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:34964 req_id = 875439d5-9b8c-4e4a-9ac1-36d899c5ffa8 response_code = 200 uri = /config
552026-07-21T18:14:34.910ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 61984, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:60178 req_id = 9df9530a-3760-4c75-b03c-cf25a94511db
562026-07-21T18:14:36.392ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 17737, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:36424 req_id = 0e310cfb-bcd1-4ead-bc05-27fabfb9eafe
572026-07-21T18:14:36.502ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40520, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:12020 req_id = a5a01df5-aad7-4582-86b3-54e10c6e801b
582026-07-21T18:14:36.524ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::a]:64337
592026-07-21T18:14:36.524ZINFOdns-server (http): request completed latency_us = 178 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:64337 req_id = 9781c58f-d72d-454f-8f19-2c3450b24f27 response_code = 200 uri = /config
602026-07-21T18:14:36.732ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::b]:41748
612026-07-21T18:14:36.732ZINFOdns-server (http): request completed latency_us = 185 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::b]:41748 req_id = 5bacc6a6-f6d3-4052-8220-0f3e5c495e09 response_code = 200 uri = /config
622026-07-21T18:14:44.911ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 27739, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:57441 req_id = dda9b409-bb18-4b15-b9c1-0b3b4aeda838
632026-07-21T18:14:54.912ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26394, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:60254 req_id = 75801c63-8510-4029-8351-8a076a2fe6e5
642026-07-21T18:15:00.174ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49781, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), ), original: Query { name: Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 98, 97, 56, 51, 55, 52, 45, 55, 101, 97, 50, 45, 52, 49, 57, 50, 45, 57, 56, 99, 97, 45, 56, 53, 48, 101, 56, 102, 53, 51, 100, 101, 49, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10." peer_addr = [fd00:1122:3344:101::7]:48780 req_id = 5d58bc85-a45a-484f-98a7-efbead59694b
652026-07-21T18:15:00.175ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49781, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), ), original: Query { name: Name("oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 98, 97, 56, 51, 55, 52, 45, 55, 101, 97, 50, 45, 52, 49, 57, 50, 45, 57, 56, 99, 97, 45, 56, 53, 48, 101, 56, 102, 53, 51, 100, 101, 49, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_efba8374-7ea2-4192-98ca-850e8f53de10." peer_addr = [fd00:1122:3344:101::7]:33194 req_id = d440f6c7-0c6a-4a05-b918-e4264ecfdc0b
662026-07-21T18:15:00.262ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52016, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), ), original: Query { name: Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 99, 55, 98, 56, 51, 51, 53, 57, 45, 48, 51, 55, 99, 45, 52, 49, 55, 100, 45, 56, 51, 102, 53, 45, 98, 55, 49, 101, 99, 53, 50, 48, 56, 54, 100, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd." peer_addr = [fd00:1122:3344:101::4]:47594 req_id = fb74063e-7816-49ea-a544-8fdb92d95d68
672026-07-21T18:15:00.263ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52016, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), ), original: Query { name: Name("oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 99, 55, 98, 56, 51, 51, 53, 57, 45, 48, 51, 55, 99, 45, 52, 49, 55, 100, 45, 56, 51, 102, 53, 45, 98, 55, 49, 101, 99, 53, 50, 48, 56, 54, 100, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_c7b83359-037c-417d-83f5-b71ec52086dd." peer_addr = [fd00:1122:3344:101::4]:39496 req_id = 65fc2d9a-1d9c-4efe-bf66-b2177f419aac
682026-07-21T18:15:00.291ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 7382, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), ), original: Query { name: Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 100, 51, 53, 102, 56, 97, 55, 45, 51, 100, 49, 98, 45, 52, 51, 52, 99, 45, 57, 102, 57, 51, 45, 99, 48, 54, 49, 55, 51, 54, 101, 100, 53, 102, 56, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8." peer_addr = [fd00:1122:3344:101::3]:44240 req_id = 8453a02c-7da5-457f-a8b6-e5c4ccf2262a
692026-07-21T18:15:00.291ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 7382, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), ), original: Query { name: Name("oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 100, 51, 53, 102, 56, 97, 55, 45, 51, 100, 49, 98, 45, 52, 51, 52, 99, 45, 57, 102, 57, 51, 45, 99, 48, 54, 49, 55, 51, 54, 101, 100, 53, 102, 56, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_dd35f8a7-3d1b-434c-9f93-c061736ed5f8." peer_addr = [fd00:1122:3344:101::3]:33414 req_id = 0e1c9d2e-4b3b-473d-996c-14ba5d3758f6
702026-07-21T18:15:00.296ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40937, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), ), original: Query { name: Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 48, 100, 100, 51, 101, 49, 55, 45, 57, 50, 55, 48, 45, 52, 53, 57, 98, 45, 98, 98, 99, 55, 45, 49, 51, 54, 101, 55, 57, 102, 99, 54, 54, 56, 54, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686." peer_addr = [fd00:1122:3344:101::5]:58138 req_id = 9b56e2df-cce5-46dc-919c-37fdf0f8ac8f
712026-07-21T18:15:00.297ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40937, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), ), original: Query { name: Name("oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 48, 100, 100, 51, 101, 49, 55, 45, 57, 50, 55, 48, 45, 52, 53, 57, 98, 45, 98, 98, 99, 55, 45, 49, 51, 54, 101, 55, 57, 102, 99, 54, 54, 56, 54, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_40dd3e17-9270-459b-bbc7-136e79fc6686." peer_addr = [fd00:1122:3344:101::5]:55195 req_id = 83f1ed36-71c2-4e43-a446-dbe5c04d8544
722026-07-21T18:15:00.313ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 64288, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5."), ), original: Query { name: Name("oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5."), query_type: A, query_class: IN, }, }, ], original: [ 44, 111, 120, 122, 95, 110, 116, 112, 95, 51, 100, 55, 97, 57, 53, 51, 101, 45, 102, 57, 98, 55, 45, 52, 48, 55, 99, 45, 97, 49, 48, 97, 45, 51, 55, 51, 50, 101, 55, 51, 53, 51, 101, 98, 53, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_ntp_3d7a953e-f9b7-407c-a10a-3732e7353eb5." peer_addr = [fd00:1122:3344:101::1b]:56808 req_id = 7311b098-e827-4152-9f11-e278fefe7cf1
732026-07-21T18:15:00.461ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26419, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:36145 req_id = 3e308f87-e34c-4593-8661-7e801b80a11f
742026-07-21T18:15:00.462ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26419, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:60430 req_id = 352509cb-dbc0-4f05-a4ae-d1147da2f0a9
752026-07-21T18:15:00.469ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18994, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), ), original: Query { name: Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 102, 53, 52, 57, 48, 51, 45, 52, 99, 57, 52, 45, 52, 102, 50, 53, 45, 98, 56, 50, 53, 45, 98, 51, 57, 100, 99, 101, 97, 100, 102, 49, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4." peer_addr = [fd00:1122:3344:101::6]:45479 req_id = d9b06b83-3989-4472-89d0-ef05a6159c10
762026-07-21T18:15:00.470ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18994, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), ), original: Query { name: Name("oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 102, 102, 53, 52, 57, 48, 51, 45, 52, 99, 57, 52, 45, 52, 102, 50, 53, 45, 98, 56, 50, 53, 45, 98, 51, 57, 100, 99, 101, 97, 100, 102, 49, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_eff54903-4c94-4f25-b825-b39dceadf1b4." peer_addr = [fd00:1122:3344:101::6]:63553 req_id = aaaa604a-12bd-4596-ac62-a56e70910732
772026-07-21T18:15:00.505ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44421, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:36379 req_id = f4e8c796-b5b2-4ea6-ac35-374e094930c6
782026-07-21T18:15:00.506ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44421, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), ), original: Query { name: Name("oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 51, 102, 100, 100, 97, 98, 102, 45, 53, 50, 53, 51, 45, 52, 57, 100, 52, 45, 56, 48, 98, 54, 45, 99, 100, 97, 99, 56, 102, 56, 98, 57, 49, 49, 101, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_a3fddabf-5253-49d4-80b6-cdac8f8b911e." peer_addr = [fd00:1122:3344:101::e]:58595 req_id = bc1a5ee6-dae6-44fd-a973-d7eaab7b1a4e