[ May 4 19:27:24 Enabled. ] [ May 4 19:27:24 Rereading configuration. ] [ May 4 19:27:24 Rereading configuration. ] [ May 4 19:27:25 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 4 19:27:25 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.904170583Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.904324396Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.926651505Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.926669996Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.926730807Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.927063264Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.969586438Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:40046"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.970733982Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"new_generation":1,"req_id":"49b0d852-6b51-41f8-8ed4-aeff73c5e91d","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.970765202Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.995141652Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.995173213Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"new_generation":1,"req_id":"49b0d852-6b51-41f8-8ed4-aeff73c5e91d","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.995185343Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"PUT","req_id":"49b0d852-6b51-41f8-8ed4-aeff73c5e91d","remote_addr":"[fd00:1122:3344:3::2]:40046","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":25034,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14072,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.033904398Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:40331","req_id":"8211c8e6-7ef5-40b6-9127-6fed45d86889","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16013,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.038419759Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:59014","req_id":"8cbffea7-a1ca-4a3c-ba97-2388c37875c2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.042221775Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:37511","req_id":"693d6b22-c8cc-432f-a689-2be42a74e9b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21763,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.672577811Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:54444","req_id":"b73aa01b-796c-4ef6-8247-1644398bc99f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64665,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.678859837Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:39067","req_id":"af55578e-3e39-4c06-b6bc-cc9c63c5843f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56937,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.67898916Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:57505","req_id":"b53d7f36-bba8-41ef-a66a-cf05d017f794","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24323,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:29.505065066Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::d]:59498","req_id":"d710c2a2-ae5c-42ec-941f-4b1035e0c908","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30285,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:39.506932936Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::d]:59357","req_id":"e3bf8d6a-fd1d-47be-94ca-d192c57f39dc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37385,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.404419527Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::a]:41701","req_id":"4ddd6fe6-170d-4e6a-bc2d-7f0121bbe7f6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19696,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.423957029Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::c]:8402","req_id":"4c0a53da-99f5-4a4d-9ae6-e0e0713a9ae9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34349,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.488854215Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::b]:46805","req_id":"b1e424dd-7fe6-4842-add7-1bfaca7d5f95","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56555,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:46.177716016Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::c]:33520","req_id":"73946997-d50d-4dae-a546-781d86ce1f6e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:46.676090371Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:41231"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:46.942296975Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"GET","req_id":"6ac814eb-9f1e-413a-9e5f-4d5927438dc2","remote_addr":"[fd00:1122:3344:101::c]:41231","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":187543,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.04844723Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:51912"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.052783457Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"new_generation":1,"req_id":"5b9a5566-985a-4992-8961-cc0e56e30029","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.052798048Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"new_generation":1,"req_id":"5b9a5566-985a-4992-8961-cc0e56e30029","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.052844589Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"PUT","req_id":"5b9a5566-985a-4992-8961-cc0e56e30029","remote_addr":"[fd00:1122:3344:101::c]:51912","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":2530,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13929,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.731572131Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::c]:22162","req_id":"2b56dd68-3f86-41da-915b-c4c6a3fc6afc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44248,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.768195157Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::e]:64768","req_id":"cdf06c93-5925-4d31-946b-1ed483f06cd9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44248,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.76884755Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::e]:57709","req_id":"7182292b-faf0-4946-aa27-9d582be8428b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.849956171Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:43238"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.850234517Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"GET","req_id":"f80f5623-a4df-49ba-b585-9fda26efd4d8","remote_addr":"[fd00:1122:3344:101::c]:43238","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54403,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:48.80142104Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::c]:44407","req_id":"1325111e-3a85-442f-8607-b444cbffc469","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:48.916968983Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:39384"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:48.917237849Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"GET","req_id":"dff058a9-9c3b-4b19-9200-76e2ae47499f","remote_addr":"[fd00:1122:3344:101::c]:39384","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28350,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:49.508986662Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::d]:53220","req_id":"8e4d30b8-8f60-49a8-bf97-9829622ea198","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38874,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:51.045365634Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::b]:24982","req_id":"e82386a8-903c-4834-867e-a27f2b663565","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1946,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:51.0511681Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::a]:60970","req_id":"11e82cbb-199a-4d3a-b667-72b84a9347bf","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.186646956Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45986"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.186910281Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"GET","req_id":"33499920-9097-4134-88e1-17c95439e8ec","remote_addr":"[fd00:1122:3344:101::a]:45986","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.270230037Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:43084"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.270511482Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"uri":"/config","method":"GET","req_id":"eee371af-4365-4f65-b530-557fe18d21c8","remote_addr":"[fd00:1122:3344:101::b]:43084","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":245,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49138,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:59.51111975Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::d]:59309","req_id":"7bce1263-a0ac-40bb-b5fc-946a09da45d7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28573,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.143307184Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::2]:47973","req_id":"d6f0abdf-2af9-41d9-b1a9-5a572f6a606e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28573,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.143862466Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::2]:63951","req_id":"a1e0770a-ea20-4b5d-8f2e-84b5a22eb7f2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27703,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 100,\n 54,\n 98,\n 56,\n 48,\n 57,\n 50,\n 100,\n 45,\n 54,\n 57,\n 57,\n 52,\n 45,\n 52,\n 99,\n 51,\n 56,\n 45,\n 57,\n 49,\n 56,\n 51,\n 45,\n 97,\n 50,\n 52,\n 50,\n 56,\n 50,\n 102,\n 100,\n 52,\n 100,\n 49,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.15731022Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::1b]:45563","req_id":"024a9066-8b32-4eb4-bba7-810f66ae4c5f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14797,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 51,\n 100,\n 55,\n 52,\n 49,\n 97,\n 45,\n 52,\n 56,\n 57,\n 57,\n 45,\n 52,\n 99,\n 48,\n 51,\n 45,\n 98,\n 102,\n 101,\n 97,\n 45,\n 55,\n 100,\n 51,\n 97,\n 54,\n 100,\n 56,\n 101,\n 52,\n 53,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.167087483Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::4]:35031","req_id":"291d122f-b684-44ec-bb79-ed4d8b88dfbe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14797,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 51,\n 100,\n 55,\n 52,\n 49,\n 97,\n 45,\n 52,\n 56,\n 57,\n 57,\n 45,\n 52,\n 99,\n 48,\n 51,\n 45,\n 98,\n 102,\n 101,\n 97,\n 45,\n 55,\n 100,\n 51,\n 97,\n 54,\n 100,\n 56,\n 101,\n 52,\n 53,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.167619864Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::4]:51902","req_id":"dbf24829-fbea-46a3-b6cc-50acbd2d1b99","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 50,\n 102,\n 54,\n 100,\n 56,\n 55,\n 99,\n 45,\n 102,\n 50,\n 102,\n 55,\n 45,\n 52,\n 102,\n 97,\n 52,\n 45,\n 57,\n 56,\n 52,\n 49,\n 45,\n 97,\n 98,\n 101,\n 52,\n 57,\n 101,\n 50,\n 101,\n 50,\n 97,\n 52,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.323692732Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::5]:42629","req_id":"8bc7814e-efc1-4c69-abd0-9b69e44108bb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 50,\n 102,\n 54,\n 100,\n 56,\n 55,\n 99,\n 45,\n 102,\n 50,\n 102,\n 55,\n 45,\n 52,\n 102,\n 97,\n 52,\n 45,\n 57,\n 56,\n 52,\n 49,\n 45,\n 97,\n 98,\n 101,\n 52,\n 57,\n 101,\n 50,\n 101,\n 50,\n 97,\n 52,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.324360866Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::5]:49208","req_id":"5966b44a-2e04-4e55-8f2c-34df8a2854a3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58126,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 53,\n 49,\n 53,\n 97,\n 101,\n 55,\n 45,\n 50,\n 49,\n 100,\n 49,\n 45,\n 52,\n 48,\n 101,\n 49,\n 45,\n 97,\n 100,\n 56,\n 55,\n 45,\n 55,\n 51,\n 55,\n 100,\n 101,\n 101,\n 52,\n 50,\n 51,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.701826678Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::6]:47360","req_id":"504fe3a0-d069-4e47-a259-a88c1d481840","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58126,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 53,\n 49,\n 53,\n 97,\n 101,\n 55,\n 45,\n 50,\n 49,\n 100,\n 49,\n 45,\n 52,\n 48,\n 101,\n 49,\n 45,\n 97,\n 100,\n 56,\n 55,\n 45,\n 55,\n 51,\n 55,\n 100,\n 101,\n 101,\n 52,\n 50,\n 51,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.702572273Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::6]:60857","req_id":"251d5373-8ccb-4418-a8eb-7b4c11f68d5f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21664,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.730936073Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::e]:59252","req_id":"bb221117-49e7-48df-84be-e7d4c086f206","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21664,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.7315402Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::e]:55026","req_id":"339f56fd-ff05-4a47-87b1-7fa3bf094137","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31393,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 55,\n 55,\n 51,\n 57,\n 56,\n 98,\n 45,\n 57,\n 101,\n 53,\n 97,\n 45,\n 52,\n 55,\n 56,\n 54,\n 45,\n 56,\n 53,\n 56,\n 52,\n 45,\n 55,\n 97,\n 55,\n 55,\n 49,\n 48,\n 97,\n 52,\n 101,\n 53,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.959714334Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::7]:63959","req_id":"9f72c0f1-34b5-4458-be38-0be54511cb95","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31393,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 55,\n 55,\n 51,\n 57,\n 56,\n 98,\n 45,\n 57,\n 101,\n 53,\n 97,\n 45,\n 52,\n 55,\n 56,\n 54,\n 45,\n 56,\n 53,\n 56,\n 52,\n 45,\n 55,\n 97,\n 55,\n 55,\n 49,\n 48,\n 97,\n 52,\n 101,\n 53,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.960278705Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::7]:49346","req_id":"fd69525b-79f1-44e4-b1af-87218491d3bb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48284,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 100,\n 49,\n 50,\n 100,\n 49,\n 99,\n 51,\n 45,\n 102,\n 48,\n 49,\n 99,\n 45,\n 52,\n 53,\n 57,\n 55,\n 45,\n 98,\n 55,\n 57,\n 50,\n 45,\n 97,\n 48,\n 53,\n 55,\n 55,\n 56,\n 49,\n 100,\n 50,\n 100,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.966639793Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::3]:61708","req_id":"be53d405-4a15-40e7-98c3-8929436af8be","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48284,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 100,\n 49,\n 50,\n 100,\n 49,\n 99,\n 51,\n 45,\n 102,\n 48,\n 49,\n 99,\n 45,\n 52,\n 53,\n 57,\n 55,\n 45,\n 98,\n 55,\n 57,\n 50,\n 45,\n 97,\n 48,\n 53,\n 55,\n 55,\n 56,\n 49,\n 100,\n 50,\n 100,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.967069172Z","hostname":"oxz_internal_dns_7b077528-ccf2-4630-b943-f4ff8027de12","pid":5440,"peer_addr":"[fd00:1122:3344:101::3]:60186","req_id":"e97d83ec-b6d4-460a-ae43-b39728a0813a","component":"dns"}