Max level shown:
1 [ Nov 3 05:57:22 Enabled. ]
2 [ Nov 3 05:57:22 Rereading configuration. ]
3 [ Nov 3 05:57:23 Rereading configuration. ]
4 [ Nov 3 05:57:24 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-addresses [fd00:1122:3344:3::1]:53 &"). ]
5 [ Nov 3 05:57:24 Method "start" exited with status 0. ]
6 Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
7 note: configured to log to "/dev/stdout"
82026-11-03T05:57:24.432ZINFOdns-server: config config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } dns_config = Config { bind_addresses: [[fd00:1122:3344:3::1]:53] }
92026-11-03T05:57:24.432ZINFOdns-server (store): opening sled database path = /data/dns
102026-11-03T05:57:24.451ZINFOdns-server (store): pruning trees for generations newer than 0
112026-11-03T05:57:24.451ZINFOdns-server (store): pruning trees for generations older than 0 keep = 3
122026-11-03T05:57:24.451ZINFOdns-server (dns): DNS server bound to address bind_address = [fd00:1122:3344:3::1]:53
132026-11-03T05:57:24.452ZINFOdns-server (http): listening local_addr = [fd00:1122:3344:3::1]:5353
142026-11-03T05:57:24.908ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 44405, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:44151 req_id = db21abdd-e09c-4bb8-b86a-a6d6a02c305d
152026-11-03T05:57:24.916ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 29211, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:20327 req_id = 6ae93632-df7f-4a6a-a2d0-d90a524880c7
162026-11-03T05:57:25.012ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12993, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:63794 req_id = ef0047e5-5e90-446d-a166-1dfb7a7c29eb
172026-11-03T05:57:25.071ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 56855, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:42618 req_id = 0f16ddc6-db43-41a2-b616-19a1ecbbd3a4
182026-11-03T05:57:25.132ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51976, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:43003 req_id = 2d25b87e-8d5a-4294-acda-e1daf608fd88
192026-11-03T05:57:25.268ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 33007, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:7021 req_id = 2740c381-f205-4718-8f53-742376cbe033
202026-11-03T05:57:25.270ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 33652, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:18806 req_id = 8b773be4-b76c-4def-8135-33b43445ff46
212026-11-03T05:57:25.305ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 42828, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:55177 req_id = 70035c11-80fd-4e1e-9641-aaadf2dfb34d
222026-11-03T05:57:25.421ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 39050, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:56368 req_id = 000bd333-c518-4f25-891c-6c1c23a31fe3
232026-11-03T05:57:25.429ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12110, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:37010 req_id = 8204b2af-36d5-4f81-99a0-5f7801ade260
242026-11-03T05:57:25.461ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 63266, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:35903 req_id = a553c4e1-6eb7-43f4-9408-e04353769b9a
252026-11-03T05:57:25.585ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 55484, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::2]:43249 req_id = 0f8e4bf3-2f49-4cf8-b234-abe1d5abf04b
262026-11-03T05:57:25.672ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:3::2]:48031
272026-11-03T05:57:25.673ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = a4b50a12-0283-4104-a5b7-c093fb9dc4a8
282026-11-03T05:57:25.673ZINFOdns-server (store): pruning trees for generations newer than 0
292026-11-03T05:57:25.694ZINFOdns-server (store): pruning trees for generations older than 1 keep = 3
302026-11-03T05:57:25.694ZINFOdns-server (store): updated generation new_generation = 1 req_id = a4b50a12-0283-4104-a5b7-c093fb9dc4a8
312026-11-03T05:57:25.694ZINFOdns-server (http): request completed latency_us = 22110 local_addr = [fd00:1122:3344:3::1]:5353 method = PUT remote_addr = [fd00:1122:3344:3::2]:48031 req_id = a4b50a12-0283-4104-a5b7-c093fb9dc4a8 response_code = 204 uri = /config
322026-11-03T05:57:38.260ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4634, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::1b]:51055 req_id = f663ee46-0114-4d7b-b43f-b737a47da87e
332026-11-03T05:57:38.264ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51686, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::1b]:58375 req_id = 6b095d79-9dc5-4c4d-a7ad-330ce487b52b
342026-11-03T05:57:38.269ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 35238, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::1b]:60438 req_id = 09217a8e-8b21-4f40-8f9f-6848397e7d1c
352026-11-03T05:59:44.334ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62995, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:57303 req_id = 783a8dea-2dc8-479f-b486-035b51028106
362026-11-03T05:59:46.097ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52424, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9."), ), original: Query { name: Name("oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 100, 102, 97, 56, 48, 100, 97, 48, 45, 56, 97, 55, 54, 45, 52, 101, 54, 97, 45, 57, 50, 52, 101, 45, 100, 56, 52, 101, 101, 55, 97, 52, 98, 102, 99, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::e]:39318 req_id = 730c05a1-6aae-47c1-a3be-66aa632f89dd
372026-11-03T05:59:46.098ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52424, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9."), ), original: Query { name: Name("oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 100, 102, 97, 56, 48, 100, 97, 48, 45, 56, 97, 55, 54, 45, 52, 101, 54, 97, 45, 57, 50, 52, 101, 45, 100, 56, 52, 101, 101, 55, 97, 52, 98, 102, 99, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_dfa80da0-8a76-4e6a-924e-d84ee7a4bfc9." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::e]:58628 req_id = 0dd65b6a-939d-4d17-984e-b4f8020424c2
382026-11-03T05:59:47.356ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 47603, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:65391 req_id = 0f543f8f-c3c3-4ead-92f5-2a79607cdb80
392026-11-03T05:59:47.889ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 54619, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:57296 req_id = 505daf7f-cc5b-4b83-83da-88f42c4c7606
402026-11-03T05:59:49.171ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4878, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:62873 req_id = ec214236-a22a-4854-b40b-b4e1b2ad7916
412026-11-03T05:59:51.366ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 34043, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:61665 req_id = 28a0161e-045a-479e-bf44-238e5b6e3b8b
422026-11-03T05:59:51.754ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 40005, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::b]:64976 req_id = d7a8f9ac-bf1b-4b9a-bb68-2f43e673fbe0
432026-11-03T05:59:53.034ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 32537, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::a]:38090 req_id = 829253c4-8a36-4e05-bb8b-00a0c474a143
442026-11-03T05:59:53.034ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 10181, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::c]:63757 req_id = e73099cf-489c-4372-b1fc-ebc82db98765
452026-11-03T05:59:53.563ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::c]:53302
462026-11-03T05:59:53.583ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::a]:52470
472026-11-03T05:59:53.605ZINFOdns-server (http): request completed latency_us = 16017 local_addr = [fd00:1122:3344:3::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:52470 req_id = ec5d28a4-0deb-427f-a3e3-fd67c5b80631 response_code = 200 uri = /config
482026-11-03T05:59:53.605ZINFOdns-server (http): request completed latency_us = 16016 local_addr = [fd00:1122:3344:3::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:53302 req_id = eff2c1dd-7159-484c-a9a3-74c1e7b89027 response_code = 200 uri = /config
492026-11-03T05:59:54.605ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 61797, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::a]:41205 req_id = 9b3d18e6-b947-4af4-865d-8dc0cd47f649
502026-11-03T05:59:54.703ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 22872, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:63504 req_id = de5886e1-8bfa-497f-a735-dfa95a4642a5
512026-11-03T05:59:54.733ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::a]:43406
522026-11-03T05:59:54.733ZINFOdns-server (http): request completed latency_us = 219 local_addr = [fd00:1122:3344:3::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:43406 req_id = e07528f5-ce08-4ab9-8d8f-c8bf9884a2ba response_code = 200 uri = /config
532026-11-03T05:59:56.061ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::a]:34977
542026-11-03T05:59:56.066ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = d279c3f9-0135-4ac8-90d6-28be212c15e1
552026-11-03T05:59:56.066ZINFOdns-server (store): updated generation new_generation = 1 req_id = d279c3f9-0135-4ac8-90d6-28be212c15e1
562026-11-03T05:59:56.066ZINFOdns-server (http): request completed latency_us = 4635 local_addr = [fd00:1122:3344:3::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::a]:34977 req_id = d279c3f9-0135-4ac8-90d6-28be212c15e1 response_code = 204 uri = /config
572026-11-03T05:59:56.078ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 35072, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::a]:63926 req_id = d70ecd3a-ed18-492c-8553-c26606e34fd7
582026-11-03T05:59:56.201ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::a]:38090
592026-11-03T05:59:56.201ZINFOdns-server (http): request completed latency_us = 211 local_addr = [fd00:1122:3344:3::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:38090 req_id = fe1cd296-df01-4bae-a1df-c2fdaac2f70e response_code = 200 uri = /config
602026-11-03T05:59:57.636ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16754, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::a]:17298 req_id = baced396-e333-4240-a9b4-aa5983555cf7
612026-11-03T05:59:57.856ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:3::1]:5353 remote_addr = [fd00:1122:3344:101::a]:59977
622026-11-03T05:59:57.856ZINFOdns-server (http): request completed latency_us = 196 local_addr = [fd00:1122:3344:3::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:59977 req_id = 7bdbab62-ef9b-4cb4-9f97-67dabf6546a8 response_code = 200 uri = /config
632026-11-03T06:00:00.269ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 55500, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977."), ), original: Query { name: Name("oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 97, 98, 52, 50, 55, 98, 50, 51, 45, 55, 100, 51, 57, 45, 52, 101, 100, 101, 45, 97, 57, 53, 53, 45, 57, 50, 99, 56, 56, 52, 54, 57, 54, 57, 55, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::4]:53453 req_id = 4b7fc80f-1b74-446a-a189-1b7cbf90206e
642026-11-03T06:00:00.270ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 55500, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977."), ), original: Query { name: Name("oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 97, 98, 52, 50, 55, 98, 50, 51, 45, 55, 100, 51, 57, 45, 52, 101, 100, 101, 45, 97, 57, 53, 53, 45, 57, 50, 99, 56, 56, 52, 54, 57, 54, 57, 55, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_ab427b23-7d39-4ede-a955-92c884696977." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::4]:46082 req_id = 42848a2a-6a0b-4f76-838c-842146a5076b
652026-11-03T06:00:00.376ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51415, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7."), ), original: Query { name: Name("oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 52, 53, 57, 56, 98, 99, 57, 45, 98, 99, 101, 49, 45, 52, 52, 100, 50, 45, 57, 97, 48, 102, 45, 55, 53, 97, 100, 50, 55, 54, 50, 101, 57, 99, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::5]:46366 req_id = f868ab4a-251f-4861-8fe1-67cfa9f57b61
662026-11-03T06:00:00.376ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 51415, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7."), ), original: Query { name: Name("oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 52, 53, 57, 56, 98, 99, 57, 45, 98, 99, 101, 49, 45, 52, 52, 100, 50, 45, 57, 97, 48, 102, 45, 55, 53, 97, 100, 50, 55, 54, 50, 101, 57, 99, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_e4598bc9-bce1-44d2-9a0f-75ad2762e9c7." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::5]:59301 req_id = 7fb9e887-cb50-4748-adec-5a7c069f36ea
672026-11-03T06:00:00.481ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 123, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::d]:56035 req_id = e54b8f01-1842-41cf-a2c6-13931905dec9
682026-11-03T06:00:00.493ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 48811, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49."), ), original: Query { name: Name("oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 102, 53, 51, 55, 51, 53, 56, 45, 49, 55, 48, 52, 45, 52, 51, 49, 57, 45, 98, 98, 100, 50, 45, 99, 100, 50, 49, 52, 51, 55, 49, 49, 102, 52, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::6]:44306 req_id = 2e9a100b-9cfd-4f92-8cb8-5d46efd34aa9
692026-11-03T06:00:00.494ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 48811, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49."), ), original: Query { name: Name("oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 102, 53, 51, 55, 51, 53, 56, 45, 49, 55, 48, 52, 45, 52, 51, 49, 57, 45, 98, 98, 100, 50, 45, 99, 100, 50, 49, 52, 51, 55, 49, 49, 102, 52, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_6f537358-1704-4319-bbd2-cd2143711f49." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::6]:46718 req_id = ecca1019-0dda-4f97-8aca-96e6bbf73a8e
702026-11-03T06:00:00.555ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60628, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529."), ), original: Query { name: Name("oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 53, 57, 53, 98, 55, 101, 57, 45, 51, 100, 56, 102, 45, 52, 56, 52, 50, 45, 56, 55, 99, 57, 45, 100, 49, 52, 97, 53, 100, 55, 97, 56, 53, 50, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::7]:37756 req_id = d7e4527c-9763-4cd4-b5dc-b603631bc92b
712026-11-03T06:00:00.555ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60628, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529."), ), original: Query { name: Name("oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 101, 53, 57, 53, 98, 55, 101, 57, 45, 51, 100, 56, 102, 45, 52, 56, 52, 50, 45, 56, 55, 99, 57, 45, 100, 49, 52, 97, 53, 100, 55, 97, 56, 53, 50, 57, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_e595b7e9-3d8f-4842-87c9-d14a5d7a8529." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::7]:49336 req_id = ef3e357a-8376-49e3-86f9-2fcec8bcadd4
722026-11-03T06:00:00.624ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26754, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1."), ), original: Query { name: Name("oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 56, 97, 51, 98, 101, 50, 101, 45, 102, 101, 51, 56, 45, 52, 57, 52, 56, 45, 57, 48, 57, 55, 45, 98, 48, 55, 98, 54, 49, 57, 52, 100, 57, 99, 49, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::3]:44970 req_id = 9d8dabce-ebef-402b-87d2-36d573a36a81
732026-11-03T06:00:00.624ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26754, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1."), ), original: Query { name: Name("oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 54, 56, 97, 51, 98, 101, 50, 101, 45, 102, 101, 51, 56, 45, 52, 57, 52, 56, 45, 57, 48, 57, 55, 45, 98, 48, 55, 98, 54, 49, 57, 52, 100, 57, 99, 49, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_68a3be2e-fe38-4948-9097-b07b6194d9c1." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::3]:65140 req_id = d77170f6-6ab6-402b-9bc8-de6f4deda5dd
742026-11-03T06:00:00.777ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 28187, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_ntp_2a69340f-b49e-4a7b-ab91-179f8e3bf59d."), ), original: Query { name: Name("oxz_ntp_2a69340f-b49e-4a7b-ab91-179f8e3bf59d."), query_type: A, query_class: IN, }, }, ], original: [ 44, 111, 120, 122, 95, 110, 116, 112, 95, 50, 97, 54, 57, 51, 52, 48, 102, 45, 98, 52, 57, 101, 45, 52, 97, 55, 98, 45, 97, 98, 57, 49, 45, 49, 55, 57, 102, 56, 101, 51, 98, 102, 53, 57, 100, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_ntp_2a69340f-b49e-4a7b-ab91-179f8e3bf59d." bind_address = [fd00:1122:3344:3::1]:53 peer_addr = [fd00:1122:3344:101::1b]:51893 req_id = 6d0b0ddb-8ce4-483e-b01e-34e35abf77e3