Max level shown:
1 [ Jul 8 00:43:38 Enabled. ]
2 [ Jul 8 00:43:38 Rereading configuration. ]
3 [ Jul 8 00:43:39 Rereading configuration. ]
4 [ Jul 8 00:43:40 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ]
5 [ Jul 8 00:43:40 Method "start" exited with status 0. ]
6 Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
7 note: configured to log to "/dev/stdout"
82026-07-08T00:43:40.552ZINFOdns-server: config config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } dns_config = Config { bind_address: [fd00:1122:3344:2::1]:53 }
92026-07-08T00:43:40.552ZINFOdns-server (store): opening sled database path = /data/dns
102026-07-08T00:43:40.574ZINFOdns-server (store): pruning trees for generations newer than 0
112026-07-08T00:43:40.574ZINFOdns-server (store): pruning trees for generations older than 0 keep = 3
122026-07-08T00:43:40.575ZINFOdns-server (dns): DNS server bound to address local_address = [fd00:1122:3344:2::1]:53
132026-07-08T00:43:40.575ZINFOdns-server (http): listening local_addr = [fd00:1122:3344:2::1]:5353
142026-07-08T00:43:40.673ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 7524, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:20121 req_id = 241bada7-4a8d-4d85-98a4-0c58a6519bde
152026-07-08T00:43:40.679ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18955, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:12260 req_id = c2ba5f27-14c2-4661-8d00-02829d1dc5af
162026-07-08T00:43:40.793ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 28193, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:31457 req_id = 3e6fc80c-4645-40eb-9e4f-6d05aa397f86
172026-07-08T00:43:40.916ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 6461, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:50632 req_id = 85b36062-8252-45e6-9dbf-d2e750bcc0d8
182026-07-08T00:43:40.961ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 19760, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:8947 req_id = e3b75208-2b30-42fb-95f2-6e7cd566e161
192026-07-08T00:43:41.027ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 39231, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:61770 req_id = 45291741-cd65-42bc-b459-0c8af66e65b5
202026-07-08T00:43:41.046ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 9501, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:45066 req_id = 9e99a1e8-dfed-45e5-9645-6408f76d2ee2
212026-07-08T00:43:41.158ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 29888, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:39046 req_id = cd632982-3765-4e8e-a633-a879275a0b30
222026-07-08T00:43:41.177ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 22310, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:19139 req_id = e40cf779-1c6d-4636-8679-6ce81a221fb4
232026-07-08T00:43:41.180ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:2::2]:55936
242026-07-08T00:43:41.181ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = c9031b56-4c4c-4a35-b5fe-4fed9abbe074
252026-07-08T00:43:41.181ZINFOdns-server (store): pruning trees for generations newer than 0
262026-07-08T00:43:41.193ZINFOdns-server (store): pruning trees for generations older than 1 keep = 3
272026-07-08T00:43:41.193ZINFOdns-server (store): updated generation new_generation = 1 req_id = c9031b56-4c4c-4a35-b5fe-4fed9abbe074
282026-07-08T00:43:41.193ZINFOdns-server (http): request completed latency_us = 12452 local_addr = [fd00:1122:3344:2::1]:5353 method = PUT remote_addr = [fd00:1122:3344:2::2]:55936 req_id = c9031b56-4c4c-4a35-b5fe-4fed9abbe074 response_code = 204 uri = /config
292026-07-08T00:43:52.903ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 34187, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:63183 req_id = f57ad8c4-1f0d-43b2-bba0-9ee9a7dc55b1
302026-07-08T00:43:52.908ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44745, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:47619 req_id = ec25a09b-18a3-434e-8a55-c73bafa7f6e2
312026-07-08T00:43:52.912ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 8726, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:60072 req_id = 3aeee892-d928-4169-9dc5-936b94ee0fa3
322026-07-08T00:44:53.402ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 38963, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:63451 req_id = 8d8dc285-09a4-4bd5-a3a5-952798369b80
332026-07-08T00:44:57.066ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 22398, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), ), original: Query { name: Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 54, 98, 98, 101, 97, 53, 57, 101, 45, 54, 100, 51, 52, 45, 52, 54, 49, 98, 45, 97, 101, 99, 99, 45, 52, 48, 102, 48, 52, 51, 49, 102, 48, 56, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4." peer_addr = [fd00:1122:3344:101::e]:63740 req_id = 7d3827b7-4545-4d89-b2a6-505be548a041
342026-07-08T00:44:57.070ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 22398, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), ), original: Query { name: Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 54, 98, 98, 101, 97, 53, 57, 101, 45, 54, 100, 51, 52, 45, 52, 54, 49, 98, 45, 97, 101, 99, 99, 45, 52, 48, 102, 48, 52, 51, 49, 102, 48, 56, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4." peer_addr = [fd00:1122:3344:101::e]:64951 req_id = 6452e92b-a704-47da-92f1-effff9b8997a
352026-07-08T00:45:00.067ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40934, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), ), original: Query { name: Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 54, 98, 98, 101, 97, 53, 57, 101, 45, 54, 100, 51, 52, 45, 52, 54, 49, 98, 45, 97, 101, 99, 99, 45, 52, 48, 102, 48, 52, 51, 49, 102, 48, 56, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4." peer_addr = [fd00:1122:3344:101::e]:42653 req_id = c0c92def-099e-4a59-8ce6-3b5f5f9a7eb4
362026-07-08T00:45:00.068ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40934, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), ), original: Query { name: Name("oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 54, 98, 98, 101, 97, 53, 57, 101, 45, 54, 100, 51, 52, 45, 52, 54, 49, 98, 45, 97, 101, 99, 99, 45, 52, 48, 102, 48, 52, 51, 49, 102, 48, 56, 98, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_6bbea59e-6d34-461b-aecc-40f0431f08b4." peer_addr = [fd00:1122:3344:101::e]:56947 req_id = 7787b8c0-4637-473a-8540-dc1ef1620c59
372026-07-08T00:45:00.203ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 25799, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc."), ), original: Query { name: Name("oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 102, 54, 101, 49, 99, 98, 56, 45, 48, 98, 53, 54, 45, 52, 49, 51, 57, 45, 57, 51, 99, 56, 45, 97, 99, 48, 51, 50, 57, 102, 101, 98, 102, 100, 99, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc." peer_addr = [fd00:1122:3344:101::6]:53908 req_id = 355c0e7e-7cf8-495a-81a4-77b1b94e6abe
382026-07-08T00:45:00.204ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 25799, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc."), ), original: Query { name: Name("oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 102, 54, 101, 49, 99, 98, 56, 45, 48, 98, 53, 54, 45, 52, 49, 51, 57, 45, 57, 51, 99, 56, 45, 97, 99, 48, 51, 50, 57, 102, 101, 98, 102, 100, 99, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_6f6e1cb8-0b56-4139-93c8-ac0329febfdc." peer_addr = [fd00:1122:3344:101::6]:51530 req_id = 5310cdea-ec16-45e6-8ab5-f3718c1c10c3
392026-07-08T00:45:00.387ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 61537, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289."), ), original: Query { name: Name("oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 49, 51, 100, 49, 55, 100, 54, 55, 45, 53, 52, 56, 98, 45, 52, 55, 52, 102, 45, 57, 54, 56, 53, 45, 56, 54, 54, 102, 100, 97, 99, 48, 98, 50, 56, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289." peer_addr = [fd00:1122:3344:101::7]:58111 req_id = c4929247-3d89-4b86-a7e6-594563ffe502
402026-07-08T00:45:00.388ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 61537, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289."), ), original: Query { name: Name("oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 49, 51, 100, 49, 55, 100, 54, 55, 45, 53, 52, 56, 98, 45, 52, 55, 52, 102, 45, 57, 54, 56, 53, 45, 56, 54, 54, 102, 100, 97, 99, 48, 98, 50, 56, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_13d17d67-548b-474f-9685-866fdac0b289." peer_addr = [fd00:1122:3344:101::7]:45621 req_id = 031004ee-fadf-43dd-b9f4-ca646a1d33ec
412026-07-08T00:45:00.409ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 820, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090."), ), original: Query { name: Name("oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 55, 55, 98, 51, 101, 49, 98, 48, 45, 100, 57, 53, 54, 45, 52, 100, 51, 53, 45, 57, 97, 101, 102, 45, 101, 56, 54, 98, 53, 100, 100, 53, 52, 48, 57, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090." peer_addr = [fd00:1122:3344:101::5]:43991 req_id = d801359f-85ff-41f3-afa8-1d72ef676db9
422026-07-08T00:45:00.409ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 820, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090."), ), original: Query { name: Name("oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 55, 55, 98, 51, 101, 49, 98, 48, 45, 100, 57, 53, 54, 45, 52, 100, 51, 53, 45, 57, 97, 101, 102, 45, 101, 56, 54, 98, 53, 100, 100, 53, 52, 48, 57, 48, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_77b3e1b0-d956-4d35-9aef-e86b5dd54090." peer_addr = [fd00:1122:3344:101::5]:43089 req_id = 82a6c844-0cf4-4d60-a1fa-7a91710c8201
432026-07-08T00:45:00.424ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16161, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584."), ), original: Query { name: Name("oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 57, 55, 54, 50, 48, 56, 54, 57, 45, 54, 98, 48, 49, 45, 52, 49, 98, 48, 45, 57, 97, 52, 101, 45, 48, 101, 97, 52, 54, 49, 53, 50, 51, 53, 56, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584." peer_addr = [fd00:1122:3344:101::3]:53519 req_id = 24c130f8-e229-4f67-9e81-9cbb0916dd4f
442026-07-08T00:45:00.425ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16161, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584."), ), original: Query { name: Name("oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 57, 55, 54, 50, 48, 56, 54, 57, 45, 54, 98, 48, 49, 45, 52, 49, 98, 48, 45, 57, 97, 52, 101, 45, 48, 101, 97, 52, 54, 49, 53, 50, 51, 53, 56, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_97620869-6b01-41b0-9a4e-0ea461523584." peer_addr = [fd00:1122:3344:101::3]:60217 req_id = a83af7f0-97ff-45f5-96b1-50c93988a0b1
452026-07-08T00:45:00.448ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60494, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02."), ), original: Query { name: Name("oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 51, 97, 102, 48, 48, 51, 100, 45, 102, 56, 48, 49, 45, 52, 51, 50, 97, 45, 56, 98, 49, 102, 45, 99, 49, 48, 98, 48, 48, 55, 48, 99, 99, 48, 50, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02." peer_addr = [fd00:1122:3344:101::4]:49694 req_id = e59bfec6-078e-46fd-917d-019e37b966cc
462026-07-08T00:45:00.449ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60494, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02."), ), original: Query { name: Name("oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 51, 97, 102, 48, 48, 51, 100, 45, 102, 56, 48, 49, 45, 52, 51, 50, 97, 45, 56, 98, 49, 102, 45, 99, 49, 48, 98, 48, 48, 55, 48, 99, 99, 48, 50, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_63af003d-f801-432a-8b1f-c10b0070cc02." peer_addr = [fd00:1122:3344:101::4]:53511 req_id = 10de8730-f968-44d6-a39b-5b15ef9cd235
472026-07-08T00:45:00.614ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4521, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_ntp_96e9d6f3-c03d-40cc-b4ea-af6f4248d7ca."), ), original: Query { name: Name("oxz_ntp_96e9d6f3-c03d-40cc-b4ea-af6f4248d7ca."), query_type: A, query_class: IN, }, }, ], original: [ 44, 111, 120, 122, 95, 110, 116, 112, 95, 57, 54, 101, 57, 100, 54, 102, 51, 45, 99, 48, 51, 100, 45, 52, 48, 99, 99, 45, 98, 52, 101, 97, 45, 97, 102, 54, 102, 52, 50, 52, 56, 100, 55, 99, 97, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_ntp_96e9d6f3-c03d-40cc-b4ea-af6f4248d7ca." peer_addr = [fd00:1122:3344:101::1b]:37231 req_id = 3573132c-2d3c-4207-aa79-e30e95e8d297
482026-07-08T00:45:01.052ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16368, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:62317 req_id = 0204a594-32ca-4e91-bb1f-e0f01d50c941
492026-07-08T00:45:01.053ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16368, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:53767 req_id = 8a8358b2-63d5-4bbc-94c1-49c3a7091a97
502026-07-08T00:45:02.252ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12945, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:39126 req_id = 36708d53-a343-4e5a-9c1b-bf89036edae4