Max level shown:
1 [ Jun 9 05:27:04 Enabled. ]
2 [ Jun 9 05:27:04 Rereading configuration. ]
3 [ Jun 9 05:27:04 Rereading configuration. ]
4 [ Jun 9 05:27:06 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ]
5 [ Jun 9 05:27:06 Method "start" exited with status 0. ]
6 Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
7 note: configured to log to "/dev/stdout"
82026-06-09T05:27:06.047ZINFOdns-server: config config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } dns_config = Config { bind_address: [fd00:1122:3344:2::1]:53 }
92026-06-09T05:27:06.048ZINFOdns-server (store): opening sled database path = /data/dns
102026-06-09T05:27:06.069ZINFOdns-server (store): pruning trees for generations newer than 0
112026-06-09T05:27:06.070ZINFOdns-server (store): pruning trees for generations older than 0 keep = 3
122026-06-09T05:27:06.070ZINFOdns-server (dns): DNS server bound to address local_address = [fd00:1122:3344:2::1]:53
132026-06-09T05:27:06.070ZINFOdns-server (http): listening local_addr = [fd00:1122:3344:2::1]:5353
142026-06-09T05:27:06.202ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 47440, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:36789 req_id = b16e008e-4182-4f0c-a77e-0cb08f085d92
152026-06-09T05:27:06.211ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62812, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:62631 req_id = 23b7b674-518b-4a0a-982b-58aabd69c4c3
162026-06-09T05:27:06.314ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37471, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:26528 req_id = a3fb302d-6d10-45d9-a1ae-d04bf6aba341
172026-06-09T05:27:06.376ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:2::2]:45751
182026-06-09T05:27:06.377ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 155bddc9-7859-4737-bbf9-c6687c4adc10
192026-06-09T05:27:06.377ZINFOdns-server (store): pruning trees for generations newer than 0
202026-06-09T05:27:06.382ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4012, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_nexus._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_nexus._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 6, 95, 110, 101, 120, 117, 115, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:51595 req_id = 11a9f5a7-00d9-4b31-8baa-87504add0cb3
212026-06-09T05:27:06.383ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 15121, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("5b80392e-15d6-4e0f-8655-e15bcab3d920.host.control-plane.oxide.internal."), ), original: Query { name: Name("5b80392e-15d6-4e0f-8655-e15bcab3d920.host.control-plane.oxide.internal."), query_type: AAAA, query_class: IN, }, }, ], original: [ 36, 53, 98, 56, 48, 51, 57, 50, 101, 45, 49, 53, 100, 54, 45, 52, 101, 48, 102, 45, 56, 54, 53, 53, 45, 101, 49, 53, 98, 99, 97, 98, 51, 100, 57, 50, 48, 4, 104, 111, 115, 116, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "5b80392e-15d6-4e0f-8655-e15bcab3d920.host.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:40241 req_id = 2551060f-6a37-4fa0-a02e-2b8c462b8d27
222026-06-09T05:27:06.383ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 43267, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("2d5872f6-24ef-4e58-be88-dbf2f316226c.host.control-plane.oxide.internal."), ), original: Query { name: Name("2d5872f6-24ef-4e58-be88-dbf2f316226c.host.control-plane.oxide.internal."), query_type: AAAA, query_class: IN, }, }, ], original: [ 36, 50, 100, 53, 56, 55, 50, 102, 54, 45, 50, 52, 101, 102, 45, 52, 101, 53, 56, 45, 98, 101, 56, 56, 45, 100, 98, 102, 50, 102, 51, 49, 54, 50, 50, 54, 99, 4, 104, 111, 115, 116, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "2d5872f6-24ef-4e58-be88-dbf2f316226c.host.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:1526 req_id = 7f5970f5-63cf-4aa5-8950-ac327b5d83ee
232026-06-09T05:27:06.383ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 16515, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("779cebba-ca00-41b9-baef-9332c5174ea9.host.control-plane.oxide.internal."), ), original: Query { name: Name("779cebba-ca00-41b9-baef-9332c5174ea9.host.control-plane.oxide.internal."), query_type: AAAA, query_class: IN, }, }, ], original: [ 36, 55, 55, 57, 99, 101, 98, 98, 97, 45, 99, 97, 48, 48, 45, 52, 49, 98, 57, 45, 98, 97, 101, 102, 45, 57, 51, 51, 50, 99, 53, 49, 55, 52, 101, 97, 57, 4, 104, 111, 115, 116, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } SERVFAIL: server is not authoritative for name: "779cebba-ca00-41b9-baef-9332c5174ea9.host.control-plane.oxide.internal." peer_addr = [fd00:1122:3344:101::2]:52344 req_id = a4ec083e-6d0e-4458-9053-d595d4a19c4b
242026-06-09T05:27:06.393ZINFOdns-server (store): pruning trees for generations older than 1 keep = 3
252026-06-09T05:27:06.393ZINFOdns-server (store): updated generation new_generation = 1 req_id = 155bddc9-7859-4737-bbf9-c6687c4adc10
262026-06-09T05:27:06.393ZINFOdns-server (http): request completed latency_us = 17043 local_addr = [fd00:1122:3344:2::1]:5353 method = PUT remote_addr = [fd00:1122:3344:2::2]:45751 req_id = 155bddc9-7859-4737-bbf9-c6687c4adc10 response_code = 204 uri = /config
272026-06-09T05:27:20.963ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 18102, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:64638 req_id = b29561f6-fd98-4c78-b013-b2c2a31e3548
282026-06-09T05:27:20.968ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 50134, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:47888 req_id = c3c82df9-6987-4703-81c6-fa944c38a1f0
292026-06-09T05:27:20.973ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 28672, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:55329 req_id = 363d3c79-53cd-4736-abfe-57f5fe2ee90e
302026-06-09T05:27:49.041ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 26938, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:41290 req_id = 46e6d2a4-9b9f-4651-9f50-08b4fdf3ec71
312026-06-09T05:27:49.045ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 2342, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:47734 req_id = 3de02a27-e5a4-4079-a3e5-879041c94e73
322026-06-09T05:27:49.050ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 52022, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:58334 req_id = ec587f3a-a18b-4036-a65a-3d34c210e168
332026-06-09T05:28:45.213ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 27520, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:54763 req_id = ffa1b1ad-1827-4933-9b32-523db1843fe7
342026-06-09T05:28:45.219ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 59642, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: AAAA, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 28, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:39334 req_id = 8c71b684-a642-4139-85b1-243fab54dbf9
352026-06-09T05:28:45.220ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41876, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("0.pool.ntp.org."), ), original: Query { name: Name("0.pool.ntp.org."), query_type: A, query_class: IN, }, }, ], original: [ 1, 48, 4, 112, 111, 111, 108, 3, 110, 116, 112, 3, 111, 114, 103, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "0.pool.ntp.org." peer_addr = [fd00:1122:3344:101::1b]:42700 req_id = 89f82a91-ee09-4c4b-af2e-c7d53555a544
362026-06-09T05:28:55.984ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 5555, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:53920 req_id = c24cffe7-61b6-4aa9-989c-7e745a3c5a5b
372026-06-09T05:28:58.918ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41332, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), ), original: Query { name: Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 101, 51, 99, 102, 49, 101, 54, 45, 51, 102, 102, 55, 45, 52, 100, 56, 48, 45, 97, 54, 101, 99, 45, 99, 49, 55, 54, 99, 51, 50, 98, 56, 50, 55, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274." peer_addr = [fd00:1122:3344:101::e]:53163 req_id = e446f644-aa2b-413b-946d-e890c307a737
382026-06-09T05:28:58.920ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 41332, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), ), original: Query { name: Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 101, 51, 99, 102, 49, 101, 54, 45, 51, 102, 102, 55, 45, 52, 100, 56, 48, 45, 97, 54, 101, 99, 45, 99, 49, 55, 54, 99, 51, 50, 98, 56, 50, 55, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274." peer_addr = [fd00:1122:3344:101::e]:61153 req_id = bed866c1-ea08-4e71-baab-0ea870afdaa6
392026-06-09T05:29:05.986ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 4142, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:64481 req_id = de90c3b7-9ad8-4299-872b-f000907dd173
402026-06-09T05:29:06.648ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 75, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:32192 req_id = a575add2-a4e5-451d-bea2-46ca70b85863
412026-06-09T05:29:06.648ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 32712, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:35255 req_id = 98ef1104-ec92-4972-acbd-da93a389dcd2
422026-06-09T05:29:07.015ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 5846, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:45616 req_id = 7216099e-bbc7-4979-bfc2-a2d81741835b
432026-06-09T05:29:15.988ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 19420, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:56785 req_id = 36e5a58d-a052-4611-9346-57f821b6c108
442026-06-09T05:29:25.279ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 15468, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:38639 req_id = f076dd7e-aad2-4d3c-a235-4ad75a2f5a46
452026-06-09T05:29:25.678ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::a]:57584
462026-06-09T05:29:25.748ZINFOdns-server (http): request completed latency_us = 43975 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:57584 req_id = 24f23d7b-5aa4-4c1c-9459-4a86dfa4cb75 response_code = 200 uri = /config
472026-06-09T05:29:25.988ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 46900, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:61647 req_id = 77e8bdaf-70e7-4914-b312-e9a8ac03388e
482026-06-09T05:29:26.387ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 34140, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:12177 req_id = eeff8898-0db7-48bd-9972-0d1c1a8356df
492026-06-09T05:29:26.414ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::a]:45702
502026-06-09T05:29:26.415ZINFOdns-server (store): attempting generation update new_generation = 1 req_id = 47dd5671-f3a3-4b94-bcf6-47a2c272020f
512026-06-09T05:29:26.415ZINFOdns-server (store): updated generation new_generation = 1 req_id = 47dd5671-f3a3-4b94-bcf6-47a2c272020f
522026-06-09T05:29:26.415ZINFOdns-server (http): request completed latency_us = 443 local_addr = [fd00:1122:3344:2::1]:5353 method = PUT remote_addr = [fd00:1122:3344:101::a]:45702 req_id = 47dd5671-f3a3-4b94-bcf6-47a2c272020f response_code = 204 uri = /config
532026-06-09T05:29:26.632ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::a]:39139
542026-06-09T05:29:26.632ZINFOdns-server (http): request completed latency_us = 198 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:39139 req_id = c5f000af-79ab-4fe9-af9e-6a4ecc901852 response_code = 200 uri = /config
552026-06-09T05:29:27.261ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 60203, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::a]:31559 req_id = d9313296-a23c-4b07-9130-da20fca5bd2f
562026-06-09T05:29:27.326ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 55623, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::b]:25548 req_id = 846c72f6-20d4-4b84-8f13-e5b44f3ef88a
572026-06-09T05:29:27.396ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 63543, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 24, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 97, 100, 109, 105, 110, 45, 107, 101, 101, 112, 101, 114, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::c]:33992 req_id = a87ee543-ad42-4f16-85b6-6fe986b7e691
582026-06-09T05:29:27.506ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::a]:55050
592026-06-09T05:29:27.506ZINFOdns-server (http): request completed latency_us = 232 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::a]:55050 req_id = 3cb315ee-2f01-4f3d-9609-066d8762ae1d response_code = 200 uri = /config
602026-06-09T05:29:27.565ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::c]:39504
612026-06-09T05:29:27.565ZINFOdns-server (http): request completed latency_us = 194 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::c]:39504 req_id = 2bf8ebb7-4007-403c-ade8-df10d1b205de response_code = 200 uri = /config
622026-06-09T05:29:27.625ZINFOdns-server (http): accepted connection local_addr = [fd00:1122:3344:2::1]:5353 remote_addr = [fd00:1122:3344:101::b]:47267
632026-06-09T05:29:27.625ZINFOdns-server (http): request completed latency_us = 210 local_addr = [fd00:1122:3344:2::1]:5353 method = GET remote_addr = [fd00:1122:3344:101::b]:47267 req_id = 8e535a7e-de44-4669-b8c0-e06cb7552905 response_code = 200 uri = /config
642026-06-09T05:29:35.990ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 12709, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:60690 req_id = a5ce3018-efda-440f-8958-2e1a90663569
652026-06-09T05:29:45.993ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 49213, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:59481 req_id = 417d3825-d582-45c6-ab6b-3e2d0a067d89
662026-06-09T05:29:55.994ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 27551, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 1, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), ), original: Query { name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN, }, }, ], original: [ 26, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 45, 99, 108, 117, 115, 116, 101, 114, 45, 110, 97, 116, 105, 118, 101, 4, 95, 116, 99, 112, 13, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 5, 111, 120, 105, 100, 101, 8, 105, 110, 116, 101, 114, 110, 97, 108, 0, 0, 33, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: Some( Edns { rcode_high: 0, version: 0, flags: EdnsFlags { dnssec_ok: false, z: 0, }, max_payload: 1232, options: OPT { options: [], }, }, ), } NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal peer_addr = [fd00:1122:3344:101::d]:52223 req_id = d98bb7e8-96e8-4bf3-8f29-0d4b94a4f8b2
672026-06-09T05:30:00.061ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 8568, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a."), ), original: Query { name: Name("oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 48, 101, 99, 49, 98, 101, 102, 49, 45, 53, 55, 54, 97, 45, 52, 48, 99, 57, 45, 97, 55, 99, 101, 45, 51, 101, 51, 52, 48, 53, 52, 54, 54, 53, 50, 97, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a." peer_addr = [fd00:1122:3344:101::6]:35514 req_id = 4ed1dbc9-d966-440e-800d-13f4ef7f1186
682026-06-09T05:30:00.062ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 8568, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a."), ), original: Query { name: Name("oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 48, 101, 99, 49, 98, 101, 102, 49, 45, 53, 55, 54, 97, 45, 52, 48, 99, 57, 45, 97, 55, 99, 101, 45, 51, 101, 51, 52, 48, 53, 52, 54, 54, 53, 50, 97, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0ec1bef1-576a-40c9-a7ce-3e340546652a." peer_addr = [fd00:1122:3344:101::6]:38755 req_id = 76f7a67f-9dbc-4144-9920-aec480872e43
692026-06-09T05:30:00.084ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 25113, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a."), ), original: Query { name: Name("oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 52, 102, 54, 50, 48, 56, 102, 45, 55, 48, 98, 101, 45, 52, 51, 54, 55, 45, 97, 50, 48, 97, 45, 57, 54, 102, 99, 50, 48, 56, 99, 98, 50, 50, 97, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a." peer_addr = [fd00:1122:3344:101::7]:39650 req_id = bf7adf3d-9dce-4a59-a2aa-fbaf9184747b
702026-06-09T05:30:00.085ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 25113, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a."), ), original: Query { name: Name("oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 100, 52, 102, 54, 50, 48, 56, 102, 45, 55, 48, 98, 101, 45, 52, 51, 54, 55, 45, 97, 50, 48, 97, 45, 57, 54, 102, 99, 50, 48, 56, 99, 98, 50, 50, 97, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_d4f6208f-70be-4367-a20a-96fc208cb22a." peer_addr = [fd00:1122:3344:101::7]:39651 req_id = 653cce67-fb0d-4b13-9c4d-2f87f0622435
712026-06-09T05:30:00.115ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 31755, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7."), ), original: Query { name: Name("oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 53, 48, 55, 55, 55, 57, 98, 45, 53, 102, 97, 101, 45, 52, 49, 49, 98, 45, 98, 98, 98, 49, 45, 101, 97, 50, 97, 51, 55, 49, 51, 55, 100, 100, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7." peer_addr = [fd00:1122:3344:101::3]:37038 req_id = 70e16562-885c-44ff-8364-7300c82b8a94
722026-06-09T05:30:00.116ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 31755, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7."), ), original: Query { name: Name("oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 52, 53, 48, 55, 55, 55, 57, 98, 45, 53, 102, 97, 101, 45, 52, 49, 49, 98, 45, 98, 98, 98, 49, 45, 101, 97, 50, 97, 51, 55, 49, 51, 55, 100, 100, 55, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_4507779b-5fae-411b-bbb1-ea2a37137dd7." peer_addr = [fd00:1122:3344:101::3]:64698 req_id = c59a8ce3-a38a-49ca-91be-e98511706059
732026-06-09T05:30:00.139ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37523, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45."), ), original: Query { name: Name("oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 50, 50, 54, 49, 48, 101, 55, 102, 45, 50, 102, 97, 102, 45, 52, 102, 97, 100, 45, 57, 49, 98, 52, 45, 56, 98, 52, 52, 52, 52, 97, 56, 54, 101, 52, 53, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45." peer_addr = [fd00:1122:3344:101::4]:34723 req_id = 1bc891cc-fce3-42a1-9c44-0131b7c08b71
742026-06-09T05:30:00.140ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 37523, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45."), ), original: Query { name: Name("oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45."), query_type: A, query_class: IN, }, }, ], original: [ 52, 111, 120, 122, 95, 99, 111, 99, 107, 114, 111, 97, 99, 104, 100, 98, 95, 50, 50, 54, 49, 48, 101, 55, 102, 45, 50, 102, 97, 102, 45, 52, 102, 97, 100, 45, 57, 49, 98, 52, 45, 56, 98, 52, 52, 52, 52, 97, 56, 54, 101, 52, 53, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_22610e7f-2faf-4fad-91b4-8b4444a86e45." peer_addr = [fd00:1122:3344:101::4]:63806 req_id = e65a0330-75ed-436d-a91d-b8d34570e89c
752026-06-09T05:30:00.403ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62560, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:55032 req_id = efddca76-a6b0-4f0c-984d-765f6520a890
762026-06-09T05:30:00.404ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 62560, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_switch."), ), original: Query { name: Name("oxz_switch."), query_type: A, query_class: IN, }, }, ], original: [ 10, 111, 120, 122, 95, 115, 119, 105, 116, 99, 104, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_switch." peer_addr = [fd00:1122:3344:101::2]:49423 req_id = bcbd978d-00a1-42fc-a415-7ac2801ea433
772026-06-09T05:30:00.578ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 63839, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_ntp_05062470-d4ac-4748-a941-ba45a3bca72c."), ), original: Query { name: Name("oxz_ntp_05062470-d4ac-4748-a941-ba45a3bca72c."), query_type: A, query_class: IN, }, }, ], original: [ 44, 111, 120, 122, 95, 110, 116, 112, 95, 48, 53, 48, 54, 50, 52, 55, 48, 45, 100, 52, 97, 99, 45, 52, 55, 52, 56, 45, 97, 57, 52, 49, 45, 98, 97, 52, 53, 97, 51, 98, 99, 97, 55, 50, 99, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_ntp_05062470-d4ac-4748-a941-ba45a3bca72c." peer_addr = [fd00:1122:3344:101::1b]:48578 req_id = 87609861-d1d2-419a-821e-c6572d6b8809
782026-06-09T05:30:00.794ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 5943, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), ), original: Query { name: Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 101, 51, 99, 102, 49, 101, 54, 45, 51, 102, 102, 55, 45, 52, 100, 56, 48, 45, 97, 54, 101, 99, 45, 99, 49, 55, 54, 99, 51, 50, 98, 56, 50, 55, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274." peer_addr = [fd00:1122:3344:101::e]:44331 req_id = d27d694d-ed4c-4dd8-b83d-517114abf12a
792026-06-09T05:30:00.795ZERROdns-server (dns): failed to handle incoming DNS message: MessageRequest { header: Header { id: 5943, message_type: Query, op_code: Query, authoritative: false, truncation: false, recursion_desired: true, recursion_available: false, authentic_data: false, checking_disabled: false, response_code: NoError, query_count: 1, answer_count: 0, name_server_count: 0, additional_count: 0, }, queries: Queries { queries: [ LowerQuery { name: LowerName( Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), ), original: Query { name: Name("oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274."), query_type: A, query_class: IN, }, }, ], original: [ 51, 111, 120, 122, 95, 99, 108, 105, 99, 107, 104, 111, 117, 115, 101, 95, 97, 101, 51, 99, 102, 49, 101, 54, 45, 51, 102, 102, 55, 45, 52, 100, 56, 48, 45, 97, 54, 101, 99, 45, 99, 49, 55, 54, 99, 51, 50, 98, 56, 50, 55, 52, 0, 0, 1, 0, 1, ], }, answers: [], name_servers: [], additionals: [], sig0: [], edns: None, } SERVFAIL: server is not authoritative for name: "oxz_clickhouse_ae3cf1e6-3ff7-4d80-a6ec-c176c32b8274." peer_addr = [fd00:1122:3344:101::e]:38149 req_id = c9d4e190-de6e-4fd3-84e4-b97835c65fc0