[ May 26 23:08:22 Enabled. ] [ May 26 23:08:22 Rereading configuration. ] [ May 26 23:08:22 Rereading configuration. ] [ May 26 23:08:24 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 26 23:08:24 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.039392106Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.039532119Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.059435574Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.059460735Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.059542647Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.059853684Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.175496254Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:53117"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.176397996Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"4ff00644-173f-46fa-a1e9-0584940fc481","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.176429517Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.194236302Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.194263372Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"4ff00644-173f-46fa-a1e9-0584940fc481","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:08:24.194287643Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"PUT","req_id":"4ff00644-173f-46fa-a1e9-0584940fc481","remote_addr":"[fd00:1122:3344:3::2]:53117","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":18340,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43384,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:08:35.319472163Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:63027","req_id":"5a542454-a92c-40af-9896-fc24a16d453a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40683,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:08:35.345658788Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:53529","req_id":"5da1db48-5fb8-48af-a0a1-a1e5f46e539c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36064,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:08:35.362121701Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:56666","req_id":"a4befc32-60d0-4dc7-9452-b749e3a24d9b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61675,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:09:53.430512925Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:61043","req_id":"5f912819-52cb-45c6-b6ef-082e846d8950","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40944,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:09:53.451410025Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:62037","req_id":"65c82996-3242-4632-a383-498048ce4fa5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11048,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:09:53.472865608Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:52395","req_id":"7d4e18c5-661c-405d-a8b7-fb173b7a58b4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41605,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:12:38.50005826Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:43737","req_id":"f971c70a-0187-43e2-b00d-cb8557c1b36b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41605,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:12:40.501141136Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:35365","req_id":"663985ff-17d4-420f-90fb-9b26dde469b6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46644,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:12:40.551869017Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:39297","req_id":"5adaa58f-61df-4e67-90f4-cd342e4a3eb5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:12:40.571609008Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:55065","req_id":"4b0dacdf-266f-441c-b852-f2e016473790","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10269,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:01.847661749Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:49851","req_id":"63c22207-a2f9-40bb-8d52-82e813449420","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59318,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 49,\n 101,\n 51,\n 56,\n 101,\n 54,\n 53,\n 45,\n 98,\n 99,\n 49,\n 52,\n 45,\n 52,\n 55,\n 102,\n 98,\n 45,\n 98,\n 100,\n 97,\n 99,\n 45,\n 101,\n 102,\n 53,\n 99,\n 54,\n 48,\n 100,\n 97,\n 101,\n 101,\n 53,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:08.1464328Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::e]:62549","req_id":"35c2fa68-517a-4357-a65c-f0ce76b33248","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59318,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 49,\n 101,\n 51,\n 56,\n 101,\n 54,\n 53,\n 45,\n 98,\n 99,\n 49,\n 52,\n 45,\n 52,\n 55,\n 102,\n 98,\n 45,\n 98,\n 100,\n 97,\n 99,\n 45,\n 101,\n 102,\n 53,\n 99,\n 54,\n 48,\n 100,\n 97,\n 101,\n 101,\n 53,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:08.158395646Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::e]:47048","req_id":"45957f1e-0500-4c42-9f75-6af5f0e2ccc2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3671,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:11.84844552Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:61025","req_id":"df6cb3bd-5e0a-428c-af82-0e951a19d43f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48675,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:18.037005032Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::a]:55921","req_id":"ccb4e9c4-60b9-4e1f-94f9-fb34a921d8dd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23764,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:18.152595931Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::c]:13622","req_id":"212d1522-3d10-479d-8541-f838c3f0d24b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57183,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:19.554721803Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::b]:64009","req_id":"d0b2e1a1-5bd6-4a18-be05-6e2aa7fdd6d0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40761,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:21.849731314Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:50250","req_id":"c7919160-f307-499f-b8b6-7949449ff2cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60612,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:31.852369144Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:63433","req_id":"54fea6bf-313a-47bc-9599-19247a8d04eb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4161,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:41.854355203Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:50922","req_id":"8bde14da-b3ad-4592-a70d-040fabd0877d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36911,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:13:51.855835673Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:50885","req_id":"91717a09-43b4-499b-94c1-ff69768ce2cb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:00.265187802Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::c]:51354","req_id":"48c2fb53-fecd-4e63-a5b1-7fc721bfe5a5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:00.76167436Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55764"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:00.839584889Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"fc94b955-2e72-4fc7-9148-ad935db8e5d0","remote_addr":"[fd00:1122:3344:101::c]:55764","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":53656,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3686,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:01.856874777Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:62287","req_id":"3c303cd3-08f6-4063-a87f-2ab08296453d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.174072115Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:45875"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.174584047Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"ddfbb8e3-03af-400e-8ab7-a23961b8cd74","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.174677049Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"ddfbb8e3-03af-400e-8ab7-a23961b8cd74","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.174684249Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"PUT","req_id":"ddfbb8e3-03af-400e-8ab7-a23961b8cd74","remote_addr":"[fd00:1122:3344:101::b]:45875","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":367,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63021,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:10.405883686Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::b]:61974","req_id":"93a5930c-1598-407a-9c59-8873ceae86a5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.689438313Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:51815"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:10.689700469Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"18576aa5-107b-46eb-97e6-a42b458ade27","remote_addr":"[fd00:1122:3344:101::b]:51815","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20843,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:11.861681496Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:61103","req_id":"b0fdf36b-50ed-4e8c-b00b-5eec6f730abc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5135,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:12.794478932Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::b]:58262","req_id":"bfd16cd9-a3c8-435b-8436-dffc91c14be0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:12.979389655Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:45356"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:12.979663451Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"96405d55-ad13-4fbf-a09e-2bf0566234ce","remote_addr":"[fd00:1122:3344:101::b]:45356","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":223,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:17.944677919Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:51836"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:17.944991137Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"3b5019ba-0936-4769-a600-5dc0146f7fe0","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:17.945007237Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"3b5019ba-0936-4769-a600-5dc0146f7fe0","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:17.945033458Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"PUT","req_id":"3b5019ba-0936-4769-a600-5dc0146f7fe0","remote_addr":"[fd00:1122:3344:101::c]:51836","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":241,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:18.088363508Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:43239"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:18.088974493Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"60518f53-80b6-4b02-b6bc-befd04fc7024","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:18.089033234Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"60518f53-80b6-4b02-b6bc-befd04fc7024","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:18.089043294Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"PUT","req_id":"60518f53-80b6-4b02-b6bc-befd04fc7024","remote_addr":"[fd00:1122:3344:101::a]:43239","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":393,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33281,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:18.92819376Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::a]:8444","req_id":"7406cb18-59bd-4ef5-93f1-6ed94db6823b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51037,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:18.933482456Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::c]:22189","req_id":"7e2f03d4-f3aa-4c0f-8b84-491222e7f792","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.072401741Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:65212"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.072826261Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"56685128-d57b-4153-8f6c-edd7e2a654dd","remote_addr":"[fd00:1122:3344:101::a]:65212","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":194,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.074311107Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:58035"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.074588133Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"49aaa916-66a9-4903-9348-3f41e98875a4","remote_addr":"[fd00:1122:3344:101::c]:58035","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":225,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.38915624Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33249"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.3895714Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"c8eb7e84-cd42-416e-8a87-5a7d9caa927f","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.389610371Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"new_generation":1,"req_id":"c8eb7e84-cd42-416e-8a87-5a7d9caa927f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:19.389617801Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"PUT","req_id":"c8eb7e84-cd42-416e-8a87-5a7d9caa927f","remote_addr":"[fd00:1122:3344:101::b]:33249","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":258,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31896,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:21.579214745Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::b]:26394","req_id":"e8720e28-16c0-4f55-a56e-d6afacde93e2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:21.674245023Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50432"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-26T23:14:21.674454468Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"uri":"/config","method":"GET","req_id":"d4986128-fe34-46c7-a8de-4cf33c6b6ea6","remote_addr":"[fd00:1122:3344:101::b]:50432","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":179,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60532,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:21.863358285Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:63411","req_id":"dbb6e818-20c6-4987-9300-d308ea90c45e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45584,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:31.864324725Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:56249","req_id":"f1c02f3f-e46f-4372-8eef-31725f33d9db","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18989,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:32.606002256Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:65421","req_id":"37b38175-a3e9-4b0e-85ec-e0551dbb790d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8203,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:32.622269354Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:63573","req_id":"46a55368-38c0-4471-8989-f8712d3785bf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37261,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:32.63930675Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:62826","req_id":"6cd4f2dc-c9b8-4e7f-840b-425fa00c62f0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10890,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:41.866141065Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:61304","req_id":"3a1e17eb-1886-47cc-a25d-e6ba7d3308e8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49616,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:14:51.867673856Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::d]:63717","req_id":"2e6e7549-ccdb-495d-9990-cb08e406ac2f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57114,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 101,\n 55,\n 53,\n 55,\n 97,\n 52,\n 57,\n 45,\n 56,\n 51,\n 49,\n 102,\n 45,\n 52,\n 102,\n 52,\n 98,\n 45,\n 98,\n 98,\n 50,\n 48,\n 45,\n 57,\n 50,\n 54,\n 52,\n 57,\n 53,\n 99,\n 102,\n 101,\n 48,\n 51,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.320358016Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::4]:37966","req_id":"76446d4e-acc3-49cd-bc7a-2616402069ee","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57114,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 101,\n 55,\n 53,\n 55,\n 97,\n 52,\n 57,\n 45,\n 56,\n 51,\n 49,\n 102,\n 45,\n 52,\n 102,\n 52,\n 98,\n 45,\n 98,\n 98,\n 50,\n 48,\n 45,\n 57,\n 50,\n 54,\n 52,\n 57,\n 53,\n 99,\n 102,\n 101,\n 48,\n 51,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3e757a49-831f-4f4b-bb20-926495cfe038.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.320896999Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::4]:36763","req_id":"91603c1c-8e98-4b28-a0a4-295311b03540","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34166,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 49,\n 101,\n 51,\n 56,\n 101,\n 54,\n 53,\n 45,\n 98,\n 99,\n 49,\n 52,\n 45,\n 52,\n 55,\n 102,\n 98,\n 45,\n 98,\n 100,\n 97,\n 99,\n 45,\n 101,\n 102,\n 53,\n 99,\n 54,\n 48,\n 100,\n 97,\n 101,\n 101,\n 53,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.323334567Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::e]:62116","req_id":"a385c7ca-7e3d-450e-97e4-2dc77a92d544","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34166,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 49,\n 101,\n 51,\n 56,\n 101,\n 54,\n 53,\n 45,\n 98,\n 99,\n 49,\n 52,\n 45,\n 52,\n 55,\n 102,\n 98,\n 45,\n 98,\n 100,\n 97,\n 99,\n 45,\n 101,\n 102,\n 53,\n 99,\n 54,\n 48,\n 100,\n 97,\n 101,\n 101,\n 53,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_d1e38e65-bc14-47fb-bdac-ef5c60daee54.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.323831949Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::e]:41831","req_id":"a245cc50-fa9b-4f20-a126-110ce4d24f6f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12042,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 102,\n 51,\n 55,\n 48,\n 101,\n 98,\n 45,\n 56,\n 56,\n 102,\n 50,\n 45,\n 52,\n 52,\n 57,\n 97,\n 45,\n 98,\n 57,\n 101,\n 97,\n 45,\n 55,\n 49,\n 100,\n 52,\n 52,\n 51,\n 57,\n 100,\n 98,\n 48,\n 53,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.383030402Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::6]:55888","req_id":"d9773e7d-d453-41f9-bf64-77bca32a44e3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12042,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 102,\n 51,\n 55,\n 48,\n 101,\n 98,\n 45,\n 56,\n 56,\n 102,\n 50,\n 45,\n 52,\n 52,\n 57,\n 97,\n 45,\n 98,\n 57,\n 101,\n 97,\n 45,\n 55,\n 49,\n 100,\n 52,\n 52,\n 51,\n 57,\n 100,\n 98,\n 48,\n 53,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c0f370eb-88f2-449a-b9ea-71d4439db059.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.383699218Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::6]:45006","req_id":"78b4b091-5691-4429-89d3-893c051aa2c2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19672,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 102,\n 48,\n 56,\n 101,\n 53,\n 99,\n 99,\n 45,\n 55,\n 52,\n 102,\n 98,\n 45,\n 52,\n 52,\n 48,\n 100,\n 45,\n 57,\n 97,\n 100,\n 57,\n 45,\n 98,\n 48,\n 55,\n 98,\n 53,\n 52,\n 53,\n 98,\n 50,\n 98,\n 100,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.430199588Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::7]:63106","req_id":"51f7bf25-9c88-42ef-8c90-6c19817a408b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19672,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 102,\n 48,\n 56,\n 101,\n 53,\n 99,\n 99,\n 45,\n 55,\n 52,\n 102,\n 98,\n 45,\n 52,\n 52,\n 48,\n 100,\n 45,\n 57,\n 97,\n 100,\n 57,\n 45,\n 98,\n 48,\n 55,\n 98,\n 53,\n 52,\n 53,\n 98,\n 50,\n 98,\n 100,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9f08e5cc-74fb-440d-9ad9-b07b545b2bdf.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.430804592Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::7]:48238","req_id":"de2848b5-9816-426f-b2f3-e967ed1d9ca6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 50,\n 54,\n 50,\n 50,\n 51,\n 51,\n 97,\n 45,\n 97,\n 56,\n 49,\n 102,\n 45,\n 52,\n 99,\n 57,\n 99,\n 45,\n 98,\n 53,\n 54,\n 57,\n 45,\n 48,\n 54,\n 54,\n 53,\n 99,\n 54,\n 99,\n 55,\n 50,\n 51,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.45166321Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::3]:41985","req_id":"beff48dc-4187-4014-9f71-a3b247b6fe7a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 50,\n 54,\n 50,\n 50,\n 51,\n 51,\n 97,\n 45,\n 97,\n 56,\n 49,\n 102,\n 45,\n 52,\n 99,\n 57,\n 99,\n 45,\n 98,\n 53,\n 54,\n 57,\n 45,\n 48,\n 54,\n 54,\n 53,\n 99,\n 54,\n 99,\n 55,\n 50,\n 51,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4262233a-a81f-4c9c-b569-0665c6c7233a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.452236883Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::3]:60156","req_id":"f058c0b6-a33c-465f-be33-39aa925f5750","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59919,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ce301fb8-5959-4907-9ce8-c54109fad3f2.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ce301fb8-5959-4907-9ce8-c54109fad3f2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 99,\n 101,\n 51,\n 48,\n 49,\n 102,\n 98,\n 56,\n 45,\n 53,\n 57,\n 53,\n 57,\n 45,\n 52,\n 57,\n 48,\n 55,\n 45,\n 57,\n 99,\n 101,\n 56,\n 45,\n 99,\n 53,\n 52,\n 49,\n 48,\n 57,\n 102,\n 97,\n 100,\n 51,\n 102,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ce301fb8-5959-4907-9ce8-c54109fad3f2.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:00.990214572Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::1b]:60481","req_id":"b41ff587-e66a-4a68-aae7-24cfb2cdf986","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4815,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 99,\n 48,\n 100,\n 97,\n 48,\n 100,\n 101,\n 45,\n 100,\n 54,\n 50,\n 98,\n 45,\n 52,\n 48,\n 101,\n 51,\n 45,\n 98,\n 53,\n 98,\n 98,\n 45,\n 48,\n 55,\n 52,\n 57,\n 50,\n 49,\n 100,\n 52,\n 102,\n 56,\n 50,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:01.022599235Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::5]:49151","req_id":"187ccd54-2c11-49c3-a243-b927e27f93ee","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4815,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 99,\n 48,\n 100,\n 97,\n 48,\n 100,\n 101,\n 45,\n 100,\n 54,\n 50,\n 98,\n 45,\n 52,\n 48,\n 101,\n 51,\n 45,\n 98,\n 53,\n 98,\n 98,\n 45,\n 48,\n 55,\n 52,\n 57,\n 50,\n 49,\n 100,\n 52,\n 102,\n 56,\n 50,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8c0da0de-d62b-40e3-b5bb-074921d4f822.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:01.023120927Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::5]:62072","req_id":"3c6ea28d-60cd-4c04-8538-ed9ada20a564","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7184,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:01.64715692Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::2]:57680","req_id":"4055f167-40c9-45f9-be88-5b34d7e91f71","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7184,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-26T23:15:01.647663862Z","hostname":"oxz_internal_dns_a0053360-4804-44c2-9ecd-a72657218291","pid":5489,"peer_addr":"[fd00:1122:3344:101::2]:62873","req_id":"9eecd8f8-f7d9-44db-b199-eb5d1e7bb13b","component":"dns"}