[ Mar 24 14:56:42 Enabled. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Mar 24 14:56:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.328866725Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.329006738Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.349161872Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.349179662Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.349232013Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.349545491Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.14748778Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:64851"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.148271958Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"9c4be483-9b2c-4540-bde5-60aed011e9ea","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.148287048Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.180620462Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.180646003Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"9c4be483-9b2c-4540-bde5-60aed011e9ea","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.180667423Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"PUT","req_id":"9c4be483-9b2c-4540-bde5-60aed011e9ea","remote_addr":"[fd00:1122:3344:3::2]:64851","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":32778,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54632,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.150099996Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:52721","req_id":"e8b2dd18-f766-4f19-840c-ced32bf7699c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14193,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.154233401Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:35494","req_id":"4986d3da-4791-4d2e-8acc-5789afa490a9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.158568011Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:41306","req_id":"e55219d0-da7e-4a11-b817-cf3e7061413c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53143,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:20.175511182Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:53626","req_id":"c77c786c-e8ae-4e40-a8b1-9d0ad638fe0e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.266794555Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::e]:35697","req_id":"f2037695-8c4f-41fb-b24f-6d47ae012b4c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.275340852Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::e]:39504","req_id":"57a3eb2b-de7c-4213-aec6-ae1266a733f7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57445,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:30.131585561Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:58899","req_id":"5bb87a24-3189-40a6-a6cb-3f6a7217b993","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61690,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:32.606230827Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::b]:60341","req_id":"0673bde4-bbf0-41da-9043-e14909d770b6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62032,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.732828305Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::a]:12329","req_id":"9b2fbdae-2cec-4d49-be03-ccbb802252cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42818,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.841100165Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::c]:40223","req_id":"2def53e5-051c-4467-9da9-f103c04aa856","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33168,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:40.1324865Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:52262","req_id":"76d509bb-5b38-4c36-849b-925e1b3ed07f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57561,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:50.133617433Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:49497","req_id":"0024565e-195a-4269-9a50-529d0faecd59","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46552,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.142240527Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:52714","req_id":"9ba318bf-6777-4a6d-b115-d021f9920a5d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.146308181Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:36548","req_id":"2aed5648-6e9e-465f-9a0f-cdf2b99c7c7f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26822,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.150710532Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:54424","req_id":"c64b35a1-c493-45f8-90c1-1df511ec75c4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5718,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:00.135229689Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:64619","req_id":"5fae1f4b-17cc-44f4-bc91-facb18e8f28b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50488,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:03.87492041Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::a]:41245","req_id":"591a4d12-541d-405a-9f13-e59135c5cd22","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.287576065Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:59202"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.354398782Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"12fac5d0-8b09-4c92-9d42-769ca7534fba","remote_addr":"[fd00:1122:3344:101::a]:59202","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":37843,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741524577Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:51175"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.744621018Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"48523404-5610-4126-8288-0442f7f84c29","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.744644458Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"48523404-5610-4126-8288-0442f7f84c29","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.744686928Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"PUT","req_id":"48523404-5610-4126-8288-0442f7f84c29","remote_addr":"[fd00:1122:3344:101::a]:51175","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":2863,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21137,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:05.073439451Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::a]:52239","req_id":"9df1c1da-f2e3-4ed4-b54d-915c3307a0d1","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.365364565Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:46337"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.365639012Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"3b217721-5127-40ee-b0a8-a5f396f2219b","remote_addr":"[fd00:1122:3344:101::a]:46337","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":201,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47253,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:06.431428829Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::a]:65285","req_id":"f4eff729-b5d1-4dcd-b878-c534c90f9706","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.554448247Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:49002"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.554680636Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"c530419f-871c-4678-9716-77810e4df640","remote_addr":"[fd00:1122:3344:101::a]:49002","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":208,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11565,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:10.138054677Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:56659","req_id":"4b3d00c9-8b00-4d2f-a502-f5786396354c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4622,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:13.446184878Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::b]:26355","req_id":"6ed18053-fafa-4e96-8e4d-87d0171d14de","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.589148067Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:47493"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.589470854Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"a6425695-0907-466f-88fa-4e4ad654d630","remote_addr":"[fd00:1122:3344:101::b]:47493","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":240,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39541,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:14.404906149Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::c]:12349","req_id":"c403f7b4-7358-404f-92b0-cd8dd9030c8c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.526530346Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:41682"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.526783462Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"c859b682-1d58-45d9-a44d-a50115e59220","remote_addr":"[fd00:1122:3344:101::c]:41682","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":207,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26586,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:20.137842029Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:51866","req_id":"8779b10b-096b-4da4-81ce-88a754fb48c1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18609,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:30.138935494Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:53209","req_id":"e51bffa0-703a-4fcc-a25d-f634d3081ab0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513679384Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:46434"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.51395284Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"112a3003-0825-4999-b1ea-a442a7f8b1b6","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513992051Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"112a3003-0825-4999-b1ea-a442a7f8b1b6","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513999191Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"PUT","req_id":"112a3003-0825-4999-b1ea-a442a7f8b1b6","remote_addr":"[fd00:1122:3344:101::b]:46434","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":213,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.623602558Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:55550"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.624048538Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"97fc0dfe-d66a-4508-bbd5-160d5fa7d0a6","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.62410945Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"97fc0dfe-d66a-4508-bbd5-160d5fa7d0a6","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.62412877Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"PUT","req_id":"97fc0dfe-d66a-4508-bbd5-160d5fa7d0a6","remote_addr":"[fd00:1122:3344:101::a]:55550","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":389,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.793756201Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:54147"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.79415006Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"7e940e97-d007-4b6d-8ac0-8a25e6bd6300","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794181841Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"new_generation":1,"req_id":"7e940e97-d007-4b6d-8ac0-8a25e6bd6300","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794214932Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"PUT","req_id":"7e940e97-d007-4b6d-8ac0-8a25e6bd6300","remote_addr":"[fd00:1122:3344:101::c]:54147","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":324,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57290,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:33.976677658Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::b]:29498","req_id":"a5436bda-726c-4256-92ae-5799bf068eb2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.21283535Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38940"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.213111437Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"d4236980-7a92-4dc2-8735-f8eddbfecc8c","remote_addr":"[fd00:1122:3344:101::b]:38940","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":239,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60965,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:34.623114138Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::c]:19042","req_id":"097a2ed2-68e9-4e6b-833d-ad5d3b40183f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.717988821Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:59371"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.718297637Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"d3441476-4237-462d-a9ca-423b57758f23","remote_addr":"[fd00:1122:3344:101::c]:59371","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":249,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40289,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:35.161686277Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::a]:62096","req_id":"c87a4557-7b89-430c-80e1-5a2897492d6b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.255277041Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63578"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.255535686Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"uri":"/config","method":"GET","req_id":"5abd8b63-ece4-4a23-92f8-0fb8e1d5bd5f","remote_addr":"[fd00:1122:3344:101::a]:63578","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8741,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:40.139495231Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:64418","req_id":"4a9cb101-70be-4287-b53d-74aa86098bd2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52204,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:50.140788862Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:64559","req_id":"4d653485-e3b4-49fe-b7a0-88f88df7c5b6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5106,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.142655134Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::d]:60248","req_id":"70411413-9010-4af2-b02f-1e5ec494bcd6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.252141212Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::7]:33699","req_id":"bd1f3c7d-68cb-48c3-bfc4-b365b62df36d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.252615773Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::7]:45448","req_id":"cb584e15-69ce-4651-964a-9a914892b8d3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.30034142Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::4]:52380","req_id":"170b4cc5-95cd-4772-9450-1e26bfccf5eb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.300873032Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::4]:47377","req_id":"7043369f-e869-4780-a6cd-333a08af0bcb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.40642381Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::3]:56841","req_id":"6b7f59e7-2093-4809-a562-a186079e7df2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.407069414Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::3]:48032","req_id":"bb04ef9e-ae22-48b4-b351-c87c3d8f9f88","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.489823737Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::e]:64660","req_id":"b662e35e-8229-466c-82e0-058b024ea973","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.490324749Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::e]:36300","req_id":"15727d7b-10c2-42ff-a9a5-a7543ce012a3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.625845275Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::6]:44222","req_id":"ade55b42-949b-464b-b125-a227873aa655","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.62648903Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::6]:38563","req_id":"56f7c93b-3033-4757-af21-29da1e1a2b87","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.637704428Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::5]:48380","req_id":"d9c976c2-9a4f-439c-b569-6e92a4d4e0c5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.638299212Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::5]:38345","req_id":"6459fb0b-bc87-480e-9cd9-f6ba1483ce8e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5721,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 102,\n 51,\n 55,\n 97,\n 52,\n 55,\n 100,\n 99,\n 45,\n 54,\n 53,\n 50,\n 54,\n 45,\n 52,\n 57,\n 57,\n 53,\n 45,\n 57,\n 54,\n 54,\n 57,\n 45,\n 57,\n 97,\n 101,\n 98,\n 51,\n 102,\n 98,\n 54,\n 48,\n 54,\n 100,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.665794544Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::1b]:40204","req_id":"f4a1169c-c41c-4c3e-a4ed-37ba1768ff61","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.857695227Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::2]:38820","req_id":"158b4e25-5dcf-48bc-b9a9-f9772db8b2e5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.85825667Z","hostname":"oxz_internal_dns_fee5262d-e151-4fa2-a8aa-685e1e5cc072","pid":5502,"peer_addr":"[fd00:1122:3344:101::2]:35184","req_id":"b57d1804-e5c9-461b-9a22-3b0ba421eb56","component":"dns"}