[ May 21 22:24:43 Enabled. ] [ May 21 22:24:43 Rereading configuration. ] [ May 21 22:24:43 Rereading configuration. ] [ May 21 22:24:45 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 21 22:24:45 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.104877471Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.104993634Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.127234515Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.127258076Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.127322098Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.127620566Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2575,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:45.21295736Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::2]:22380","req_id":"507be86d-6663-48f9-8b5a-84e27579439a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39299,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:45.373064652Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::2]:35970","req_id":"81eca348-293d-440b-bb57-a9cd6b63c5e2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37742,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:45.719978082Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::2]:17423","req_id":"7b9f3452-ed5f-42be-8b9c-831631b4dcdc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.884158818Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:56798"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.884873438Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"0f859002-a5c5-4077-b4a7-7492ce5473d4","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.884887708Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.896908394Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.896939335Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"0f859002-a5c5-4077-b4a7-7492ce5473d4","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:24:45.896946405Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"PUT","req_id":"0f859002-a5c5-4077-b4a7-7492ce5473d4","remote_addr":"[fd00:1122:3344:3::2]:56798","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":12444,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63568,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:56.841454737Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:49863","req_id":"bfa9266b-7992-44f5-8ec3-0197170829d1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26811,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:56.854939143Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:59647","req_id":"a1c48612-5f30-4af0-aed7-b3fdb166a104","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13994,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:24:56.873367108Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:39685","req_id":"0fb364c8-26f3-45c5-88e6-750ada4dd515","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23813,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:26:21.894951602Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:42066","req_id":"a164866a-95f0-4d08-bf0a-8688f5ecb586","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 368,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:26:21.911786406Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:60266","req_id":"d14737f4-bbfd-48b7-ad71-a9da820cae42","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54458,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:26:21.92869085Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:49006","req_id":"4402aced-9cc7-4d49-9d18-0dc89fe7e10c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3137,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:06.257864763Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:57802","req_id":"8e528604-a473-4f10-90b3-6185fabe3883","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2317,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 102,\n 100,\n 102,\n 54,\n 100,\n 55,\n 52,\n 45,\n 50,\n 97,\n 48,\n 57,\n 45,\n 52,\n 56,\n 50,\n 99,\n 45,\n 57,\n 101,\n 97,\n 48,\n 45,\n 54,\n 48,\n 57,\n 54,\n 100,\n 53,\n 99,\n 56,\n 57,\n 51,\n 97,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:10.772050815Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::e]:39446","req_id":"c3eb192a-bf14-4416-bd57-332edb86ba05","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2317,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 102,\n 100,\n 102,\n 54,\n 100,\n 55,\n 52,\n 45,\n 50,\n 97,\n 48,\n 57,\n 45,\n 52,\n 56,\n 50,\n 99,\n 45,\n 57,\n 101,\n 97,\n 48,\n 45,\n 54,\n 48,\n 57,\n 54,\n 100,\n 53,\n 99,\n 56,\n 57,\n 51,\n 97,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:10.773037293Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::e]:56147","req_id":"058d2c72-8ed9-4322-9bf5-198b31ef5825","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38088,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:15.790241988Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::b]:48999","req_id":"79db3022-36a6-46ed-bcac-2cea2ff29da3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21233,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:15.792173832Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::a]:39305","req_id":"4a066b45-5523-4b12-86a2-55cfac35cb49","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26534,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:15.793148019Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::c]:18104","req_id":"4a6a1819-b177-4043-b011-b516b63b94f8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41503,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:16.23246423Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:51411","req_id":"75b82c4f-8cfd-491c-8c46-3ae588be1adb","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.344345008Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:48332"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.396188206Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"9499d4fe-fec3-4e98-b505-025489ec3154","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.396266208Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"9499d4fe-fec3-4e98-b505-025489ec3154","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.396283018Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"PUT","req_id":"9499d4fe-fec3-4e98-b505-025489ec3154","remote_addr":"[fd00:1122:3344:101::a]:48332","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":24342,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.714144237Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:39908"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.714269181Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:61123"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.714293281Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55115"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.726009378Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"071b9490-2d89-4950-9315-4decb523328d","remote_addr":"[fd00:1122:3344:101::a]:39908","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":11190,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.726085811Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"56227fb5-47f1-47b8-b897-360af3c6b79c","remote_addr":"[fd00:1122:3344:101::b]:61123","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":11754,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:17.726135252Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"2a256906-e671-4a41-95f1-6b3cd06b48d9","remote_addr":"[fd00:1122:3344:101::c]:55115","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":11727,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4249,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:18.589641212Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::a]:28642","req_id":"e1c94fa6-2f63-4735-8f99-450b32b19bde","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:18.724078587Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:50713"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:18.724305514Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"9a953997-f61d-4fa5-822c-b45f672502d1","remote_addr":"[fd00:1122:3344:101::a]:50713","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":185,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22491,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:19.098861336Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::a]:10425","req_id":"cb78174f-c2b1-4910-aec6-2efabecf5d98","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:19.242884079Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:52419"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:19.243132086Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"2307a9c8-e7d0-418d-b6fc-6fc525523a82","remote_addr":"[fd00:1122:3344:101::a]:52419","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21842,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:19.587910746Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::a]:5067","req_id":"9e3eb917-2e08-411c-ae72-7a13154c3baa","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:19.712180358Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:50098"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:19.712675022Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"fb3e1a50-5ec8-486e-a02c-526bac38ffe9","remote_addr":"[fd00:1122:3344:101::a]:50098","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":243,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1146,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:25.353518055Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:63628","req_id":"b1d40274-8e7f-4b3d-962a-bd1e95864cea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39382,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:25.366862158Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:56901","req_id":"1fd8fe34-c698-4adb-8bf1-c5c6bbdd8299","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:25.383150293Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:38566","req_id":"bee47fea-e564-4895-ae09-aca1ccc59012","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46501,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:26.233923857Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:65365","req_id":"220c577e-fe8f-4088-a290-d1e3811543cb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22865,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:26.249641796Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::c]:51947","req_id":"261f5b6c-9ec5-4f58-9eb4-4954984f1696","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21083,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:26.253243377Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::b]:55744","req_id":"a6beb14b-2a6a-414a-8eaf-5b3391dfa5b5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:26.382506248Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:61847"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:26.382748034Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"413f07f2-9786-45c4-9f48-a039c84f91dd","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":195,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:26.432519505Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:40763"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:28:26.432766671Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"b1ecfb30-3701-4b29-af7d-201fa622345f","remote_addr":"[fd00:1122:3344:101::c]:40763","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":186,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60998,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:36.235872634Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:50815","req_id":"e93a2f7c-beff-4b51-8484-59b82e09d3d1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44114,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:46.237604535Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:58344","req_id":"c2635cdd-d15d-48ad-8c9a-e6b455e4267f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6529,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:28:56.239334165Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:62092","req_id":"9f0e964c-4575-4949-8587-76ff0b9d561b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62545,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:06.239800471Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:51382","req_id":"eba29442-8089-41ab-97c0-f136ba81270e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.462745352Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33132"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.464502381Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"8ce11263-7365-43bc-bf86-2ef7df676bfb","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.464540982Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"8ce11263-7365-43bc-bf86-2ef7df676bfb","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.464560762Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"PUT","req_id":"8ce11263-7365-43bc-bf86-2ef7df676bfb","remote_addr":"[fd00:1122:3344:101::a]:33132","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":206,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.809485477Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:50353"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.809892788Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"6247e04f-a007-436a-bd1c-d4c8747cb0eb","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.80994204Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"6247e04f-a007-436a-bd1c-d4c8747cb0eb","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.809975481Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"PUT","req_id":"6247e04f-a007-436a-bd1c-d4c8747cb0eb","remote_addr":"[fd00:1122:3344:101::c]:50353","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":294,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.885426968Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33150"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.885743927Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"2640a28b-9854-42e5-a549-868e08cda53e","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.885792768Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"new_generation":1,"req_id":"2640a28b-9854-42e5-a549-868e08cda53e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:15.885805529Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"PUT","req_id":"2640a28b-9854-42e5-a549-868e08cda53e","remote_addr":"[fd00:1122:3344:101::b]:33150","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":297,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54597,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:16.241874705Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:55102","req_id":"f9a3ccf6-1f7e-4e96-89fc-c4a1d700809f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50066,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:16.798850082Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::a]:55575","req_id":"6abaa874-ffda-471b-8144-2c6410ab925c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27237,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:16.860798133Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::b]:3468","req_id":"ff8b33c1-5992-40bf-af71-303f2647c4ac","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 796,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:16.87288596Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::c]:60029","req_id":"5d2d27ff-6f30-4de4-b57a-5b5f95e231dd","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:16.976175955Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:59536"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:16.976437633Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"5b3c215a-e975-4263-aeca-3628a1356fc1","remote_addr":"[fd00:1122:3344:101::b]:59536","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":216,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:17.013148148Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:43869"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:17.013381955Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"7f99f843-ccd0-4ce1-8891-8249a53328ca","remote_addr":"[fd00:1122:3344:101::c]:43869","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":197,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:17.047870248Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:50577"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-21T22:29:17.048096734Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"uri":"/config","method":"GET","req_id":"aaf61475-5587-4aea-a7c9-8886696f4418","remote_addr":"[fd00:1122:3344:101::a]:50577","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":185,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52841,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:26.243815615Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:65168","req_id":"1e781160-6727-4819-9832-507b2cc913f6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1167,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:36.24509158Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:59116","req_id":"cb5a20ee-f484-4eb5-bd00-527295d0ced9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34161,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:46.246670573Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:61659","req_id":"b76d2ef8-a841-4516-aba2-0d37a190ad20","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53102,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:29:56.247258587Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::d]:61123","req_id":"29278c95-4739-4d9b-a5a8-608bc169784b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62481,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 102,\n 100,\n 102,\n 54,\n 100,\n 55,\n 52,\n 45,\n 50,\n 97,\n 48,\n 57,\n 45,\n 52,\n 56,\n 50,\n 99,\n 45,\n 57,\n 101,\n 97,\n 48,\n 45,\n 54,\n 48,\n 57,\n 54,\n 100,\n 53,\n 99,\n 56,\n 57,\n 51,\n 97,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.498239382Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::e]:49133","req_id":"eacec3c6-76b8-43cc-85a3-4bddd0180180","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62481,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 102,\n 100,\n 102,\n 54,\n 100,\n 55,\n 52,\n 45,\n 50,\n 97,\n 48,\n 57,\n 45,\n 52,\n 56,\n 50,\n 99,\n 45,\n 57,\n 101,\n 97,\n 48,\n 45,\n 54,\n 48,\n 57,\n 54,\n 100,\n 53,\n 99,\n 56,\n 57,\n 51,\n 97,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4fdf6d74-2a09-482c-9ea0-6096d5c893a0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.498824608Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::e]:42510","req_id":"7273e10e-9323-43de-88be-99c1bcf550ff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20433,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_17fb61ad-c1f7-4f92-9d13-b409ba830caf.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_17fb61ad-c1f7-4f92-9d13-b409ba830caf.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 49,\n 55,\n 102,\n 98,\n 54,\n 49,\n 97,\n 100,\n 45,\n 99,\n 49,\n 102,\n 55,\n 45,\n 52,\n 102,\n 57,\n 50,\n 45,\n 57,\n 100,\n 49,\n 51,\n 45,\n 98,\n 52,\n 48,\n 57,\n 98,\n 97,\n 56,\n 51,\n 48,\n 99,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_17fb61ad-c1f7-4f92-9d13-b409ba830caf.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.562688332Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::1b]:57614","req_id":"ff05e7e8-8184-4b7f-bff4-0935f92c7bc1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60326,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 97,\n 101,\n 55,\n 54,\n 98,\n 57,\n 54,\n 45,\n 101,\n 55,\n 51,\n 98,\n 45,\n 52,\n 99,\n 97,\n 99,\n 45,\n 97,\n 57,\n 54,\n 49,\n 45,\n 51,\n 54,\n 99,\n 101,\n 50,\n 55,\n 56,\n 53,\n 101,\n 57,\n 55,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.577756233Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::6]:46716","req_id":"3c56594d-5c3c-488d-9017-dedfb3f9b377","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60326,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 97,\n 101,\n 55,\n 54,\n 98,\n 57,\n 54,\n 45,\n 101,\n 55,\n 51,\n 98,\n 45,\n 52,\n 99,\n 97,\n 99,\n 45,\n 97,\n 57,\n 54,\n 49,\n 45,\n 51,\n 54,\n 99,\n 101,\n 50,\n 55,\n 56,\n 53,\n 101,\n 57,\n 55,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5ae76b96-e73b-4cac-a961-36ce2785e971.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.578415992Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::6]:34128","req_id":"1d383a41-51ea-4216-b7f7-32db3b57dc43","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3230,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 51,\n 51,\n 97,\n 100,\n 98,\n 48,\n 55,\n 45,\n 97,\n 57,\n 56,\n 99,\n 45,\n 52,\n 99,\n 51,\n 54,\n 45,\n 57,\n 55,\n 102,\n 55,\n 45,\n 57,\n 101,\n 102,\n 53,\n 53,\n 97,\n 54,\n 55,\n 100,\n 101,\n 98,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.618606944Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::4]:50400","req_id":"8afa6a38-3e0e-48c3-81d3-c5429d90110e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3230,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 51,\n 51,\n 97,\n 100,\n 98,\n 48,\n 55,\n 45,\n 97,\n 57,\n 56,\n 99,\n 45,\n 52,\n 99,\n 51,\n 54,\n 45,\n 57,\n 55,\n 102,\n 55,\n 45,\n 57,\n 101,\n 102,\n 53,\n 53,\n 97,\n 54,\n 55,\n 100,\n 101,\n 98,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_333adb07-a98c-4c36-97f7-9ef55a67deb9.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.619036216Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::4]:54492","req_id":"1c27b391-eb78-4fcc-81cc-5c8994f78389","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14789,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 50,\n 52,\n 52,\n 99,\n 102,\n 54,\n 48,\n 45,\n 57,\n 100,\n 55,\n 53,\n 45,\n 52,\n 52,\n 50,\n 53,\n 45,\n 98,\n 48,\n 101,\n 101,\n 45,\n 48,\n 49,\n 98,\n 100,\n 49,\n 52,\n 48,\n 99,\n 101,\n 97,\n 53,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.646822522Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::3]:42450","req_id":"6c7e0e50-1eac-4e08-81cf-62fffb276a58","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14789,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 50,\n 52,\n 52,\n 99,\n 102,\n 54,\n 48,\n 45,\n 57,\n 100,\n 55,\n 53,\n 45,\n 52,\n 52,\n 50,\n 53,\n 45,\n 98,\n 48,\n 101,\n 101,\n 45,\n 48,\n 49,\n 98,\n 100,\n 49,\n 52,\n 48,\n 99,\n 101,\n 97,\n 53,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8244cf60-9d75-4425-b0ee-01bd140cea5c.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.647276935Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::3]:41529","req_id":"f95b9e1d-f7e7-4ae2-925b-50137e0c1d7b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62486,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 99,\n 57,\n 101,\n 97,\n 97,\n 97,\n 55,\n 45,\n 98,\n 53,\n 56,\n 100,\n 45,\n 52,\n 55,\n 99,\n 48,\n 45,\n 98,\n 53,\n 55,\n 55,\n 45,\n 49,\n 99,\n 48,\n 50,\n 97,\n 48,\n 102,\n 101,\n 55,\n 102,\n 51,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.678818526Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::5]:36656","req_id":"ef43f296-32c2-43cc-a7c2-9c6eec03c270","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62486,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 99,\n 57,\n 101,\n 97,\n 97,\n 97,\n 55,\n 45,\n 98,\n 53,\n 56,\n 100,\n 45,\n 52,\n 55,\n 99,\n 48,\n 45,\n 98,\n 53,\n 55,\n 55,\n 45,\n 49,\n 99,\n 48,\n 50,\n 97,\n 48,\n 102,\n 101,\n 55,\n 102,\n 51,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_cc9eaaa7-b58d-47c0-b577-1c02a0fe7f3b.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.679480184Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::5]:60648","req_id":"de89d2e7-dc21-46a9-b60b-77c493f24acd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30373,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 51,\n 97,\n 55,\n 100,\n 99,\n 102,\n 56,\n 45,\n 53,\n 48,\n 99,\n 56,\n 45,\n 52,\n 53,\n 102,\n 50,\n 45,\n 97,\n 57,\n 49,\n 98,\n 45,\n 53,\n 98,\n 53,\n 51,\n 101,\n 51,\n 101,\n 99,\n 101,\n 54,\n 97,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.683179338Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::7]:58223","req_id":"b6dc4f3f-5ae4-4989-8bda-a3b0d65c11f6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30373,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 51,\n 97,\n 55,\n 100,\n 99,\n 102,\n 56,\n 45,\n 53,\n 48,\n 99,\n 56,\n 45,\n 52,\n 53,\n 102,\n 50,\n 45,\n 97,\n 57,\n 49,\n 98,\n 45,\n 53,\n 98,\n 53,\n 51,\n 101,\n 51,\n 101,\n 99,\n 101,\n 54,\n 97,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_63a7dcf8-50c8-45f2-a91b-5b53e3ece6a6.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:00.683799385Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::7]:42245","req_id":"9702bb28-778d-4c96-abfd-1d553fddb76a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20422,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:01.329591003Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::2]:53175","req_id":"4b707679-b348-48dd-9dc3-2d436d7ee6bc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20422,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-21T22:30:01.330128138Z","hostname":"oxz_internal_dns_77b0069c-45b2-402d-a88a-c46fe5f0e7d0","pid":5428,"peer_addr":"[fd00:1122:3344:101::2]:40388","req_id":"4b8ac0a9-919d-4361-b415-0e4889d100f7","component":"dns"}