[ May 4 19:27:24 Enabled. ] [ May 4 19:27:24 Rereading configuration. ] [ May 4 19:27:24 Rereading configuration. ] [ May 4 19:27:25 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ May 4 19:27:25 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.891685542Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.891850045Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.913142204Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.913158614Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.913211365Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.91347126Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35326,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:25.922231937Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:62325","req_id":"781ed60c-1c18-4759-843b-c2524263f5b2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.996904048Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:50449"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.997647283Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"new_generation":1,"req_id":"55e0aa43-609a-4247-84a3-1177939872df","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:25.997666913Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"store"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9146,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:26.001552101Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:14757","req_id":"dcbbb881-7666-4ec5-aed0-9e5e98ae02ae","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16737,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"6a0611e6-38d2-4444-bc5c-513ef79356de.host.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"6a0611e6-38d2-4444-bc5c-513ef79356de.host.control-plane.oxide.internal.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 36,\n 54,\n 97,\n 48,\n 54,\n 49,\n 49,\n 101,\n 54,\n 45,\n 51,\n 56,\n 100,\n 50,\n 45,\n 52,\n 52,\n 52,\n 52,\n 45,\n 98,\n 99,\n 53,\n 99,\n 45,\n 53,\n 49,\n 51,\n 101,\n 102,\n 55,\n 57,\n 51,\n 53,\n 54,\n 100,\n 101,\n 4,\n 104,\n 111,\n 115,\n 116,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"6a0611e6-38d2-4444-bc5c-513ef79356de.host.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:26.00299062Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:48876","req_id":"c7a26e39-5392-4a42-9c2a-2f15d3303adc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9868,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"f89597bb-ad49-4d62-8cd6-c4c977aaf9e2.host.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"f89597bb-ad49-4d62-8cd6-c4c977aaf9e2.host.control-plane.oxide.internal.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 36,\n 102,\n 56,\n 57,\n 53,\n 57,\n 55,\n 98,\n 98,\n 45,\n 97,\n 100,\n 52,\n 57,\n 45,\n 52,\n 100,\n 54,\n 50,\n 45,\n 56,\n 99,\n 100,\n 54,\n 45,\n 99,\n 52,\n 99,\n 57,\n 55,\n 55,\n 97,\n 97,\n 102,\n 57,\n 101,\n 50,\n 4,\n 104,\n 111,\n 115,\n 116,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"f89597bb-ad49-4d62-8cd6-c4c977aaf9e2.host.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:26.003042181Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:58004","req_id":"f3b30de1-8088-450f-a3a1-abd4921a62ec","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46662,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"88b09bf5-c38f-4ac8-892e-55b03757b0ae.host.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"88b09bf5-c38f-4ac8-892e-55b03757b0ae.host.control-plane.oxide.internal.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 36,\n 56,\n 56,\n 98,\n 48,\n 57,\n 98,\n 102,\n 53,\n 45,\n 99,\n 51,\n 56,\n 102,\n 45,\n 52,\n 97,\n 99,\n 56,\n 45,\n 56,\n 57,\n 50,\n 101,\n 45,\n 53,\n 53,\n 98,\n 48,\n 51,\n 55,\n 53,\n 55,\n 98,\n 48,\n 97,\n 101,\n 4,\n 104,\n 111,\n 115,\n 116,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"88b09bf5-c38f-4ac8-892e-55b03757b0ae.host.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:26.003083122Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:37150","req_id":"9f944f0e-afba-4d54-95cd-2993117dfbbf","component":"dns"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:26.020103115Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:26.020118955Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"new_generation":1,"req_id":"55e0aa43-609a-4247-84a3-1177939872df","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:27:26.020159225Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"PUT","req_id":"55e0aa43-609a-4247-84a3-1177939872df","remote_addr":"[fd00:1122:3344:1::2]:50449","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":22873,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14072,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.033411568Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:65127","req_id":"d609240a-efff-46d7-9798-a911f30197fa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16013,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.03799255Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:54347","req_id":"21319f8a-7a89-4225-92bf-551fbe719259","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:27:37.041939209Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:50898","req_id":"7fa90455-25c6-4d77-bc73-8ecffa0c2cf6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21763,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.353807918Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:45686","req_id":"1f6f8b91-fc40-40b7-86d0-01e9666fec65","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64665,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.467903343Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:47338","req_id":"e368f853-7960-4448-9004-b680bd8bb747","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56937,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:16.471900743Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:50921","req_id":"82c62faa-232d-4bd0-bd7a-068bc1021460","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15733,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:29.50528623Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:63895","req_id":"cff1a8f8-dd09-46ad-a4f9-a35e3fd183bd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28091,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:39.506948596Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:65466","req_id":"cf16936e-03f2-4d67-901a-692fb1bc2164","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58673,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.404395216Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::a]:3392","req_id":"397c782f-6d10-4042-af1a-95fc883a4ee0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.423954329Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:13074","req_id":"e9552c69-2333-414f-a6d1-0d3eeaa5fdf3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57250,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:40.488829934Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:1866","req_id":"0140eca9-f61c-4aba-8c45-b49f475cb32a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37236,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:46.177698306Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:19537","req_id":"a03c109f-96e3-40ff-8002-dd99de3568cd","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:46.981430592Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52936"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.049266586Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:46329"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.130453739Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"new_generation":1,"req_id":"84643c0a-5441-4e43-8624-c9033b0c6efe","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.130638883Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"new_generation":1,"req_id":"84643c0a-5441-4e43-8624-c9033b0c6efe","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.133933989Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"PUT","req_id":"84643c0a-5441-4e43-8624-c9033b0c6efe","remote_addr":"[fd00:1122:3344:101::c]:46329","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":64859,"response_code":204} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.138067713Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"GET","req_id":"df9d23bc-882d-4928-a196-2d0e0d2c6255","remote_addr":"[fd00:1122:3344:101::c]:52936","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":68998,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59886,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.731580141Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:54947","req_id":"7e24596b-2d08-4b66-972d-99c6284c1edd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44248,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.767548314Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:37092","req_id":"089a3a1f-f33a-46ba-9226-542609345da8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44248,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:47.768432102Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:51893","req_id":"274b8b68-7e87-4e5f-a1ef-64876e0d255c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.850863599Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:34167"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:47.851106544Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"GET","req_id":"b6654be1-ac42-4c20-a5c9-5b2e1b5d1a54","remote_addr":"[fd00:1122:3344:101::c]:34167","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":196,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51013,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:48.80142549Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:1572","req_id":"f566760b-4440-4277-a980-7b1dfbcba75b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:48.917672347Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:40426"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:48.917898212Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"GET","req_id":"f4349569-8bdf-4643-b512-67054146166c","remote_addr":"[fd00:1122:3344:101::c]:40426","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":197,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20740,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:49.50891473Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:61815","req_id":"5adcd971-661d-492f-b389-d786aa3bfa9c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29897,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:51.045343613Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:42273","req_id":"b7b25f30-1d5d-43a1-8623-88c8adaef802","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17997,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:51.051168091Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::a]:52591","req_id":"60905325-6ec8-4abf-b1fe-f2b3f49d9724","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.187476552Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:48908"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.187793619Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"GET","req_id":"64808b8d-aa84-44dd-bb35-397e8aae624d","remote_addr":"[fd00:1122:3344:101::a]:48908","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":265,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.270998482Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:57769"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-04T19:29:51.271240727Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"uri":"/config","method":"GET","req_id":"735899ab-f810-40f2-bdce-91c390fd55f5","remote_addr":"[fd00:1122:3344:101::b]:57769","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":197,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14220,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:29:59.51111996Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:53451","req_id":"f3c0ab82-7755-4f07-9a6b-7ac18d8429bc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28573,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.142807794Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:50320","req_id":"b25d4c93-8289-4a89-aaae-b403d4c1cf40","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28573,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.143517939Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:46504","req_id":"26b0279e-997d-4fa9-b4eb-f256e9b52e03","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27703,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 100,\n 54,\n 98,\n 56,\n 48,\n 57,\n 50,\n 100,\n 45,\n 54,\n 57,\n 57,\n 52,\n 45,\n 52,\n 99,\n 51,\n 56,\n 45,\n 57,\n 49,\n 56,\n 51,\n 45,\n 97,\n 50,\n 52,\n 50,\n 56,\n 50,\n 102,\n 100,\n 52,\n 100,\n 49,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_d6b8092d-6994-4c38-9183-a24282fd4d1f.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.156868807Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:44615","req_id":"7086d126-d6ad-4448-a16d-3c1fa69a7cc6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14797,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 51,\n 100,\n 55,\n 52,\n 49,\n 97,\n 45,\n 52,\n 56,\n 57,\n 57,\n 45,\n 52,\n 99,\n 48,\n 51,\n 45,\n 98,\n 102,\n 101,\n 97,\n 45,\n 55,\n 100,\n 51,\n 97,\n 54,\n 100,\n 56,\n 101,\n 52,\n 53,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.166515131Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::4]:34863","req_id":"8a6972a1-ef81-4faf-933f-69ca1e3fe040","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14797,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 51,\n 100,\n 55,\n 52,\n 49,\n 97,\n 45,\n 52,\n 56,\n 57,\n 57,\n 45,\n 52,\n 99,\n 48,\n 51,\n 45,\n 98,\n 102,\n 101,\n 97,\n 45,\n 55,\n 100,\n 51,\n 97,\n 54,\n 100,\n 56,\n 101,\n 52,\n 53,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e43d741a-4899-4c03-bfea-7d3a6d8e45dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.167245886Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::4]:46591","req_id":"b54d109e-0f38-41db-9a85-0232c7194351","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 50,\n 102,\n 54,\n 100,\n 56,\n 55,\n 99,\n 45,\n 102,\n 50,\n 102,\n 55,\n 45,\n 52,\n 102,\n 97,\n 52,\n 45,\n 57,\n 56,\n 52,\n 49,\n 45,\n 97,\n 98,\n 101,\n 52,\n 57,\n 101,\n 50,\n 101,\n 50,\n 97,\n 52,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.323206863Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::5]:51212","req_id":"0a6a7cc2-d21f-48c5-b50a-c8612d73a978","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 50,\n 102,\n 54,\n 100,\n 56,\n 55,\n 99,\n 45,\n 102,\n 50,\n 102,\n 55,\n 45,\n 52,\n 102,\n 97,\n 52,\n 45,\n 57,\n 56,\n 52,\n 49,\n 45,\n 97,\n 98,\n 101,\n 52,\n 57,\n 101,\n 50,\n 101,\n 50,\n 97,\n 52,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a2f6d87c-f2f7-4fa4-9841-abe49e2e2a4a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.323891056Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::5]:45406","req_id":"de291be2-32c1-4130-af47-fb4f8bb58c22","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58126,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 53,\n 49,\n 53,\n 97,\n 101,\n 55,\n 45,\n 50,\n 49,\n 100,\n 49,\n 45,\n 52,\n 48,\n 101,\n 49,\n 45,\n 97,\n 100,\n 56,\n 55,\n 45,\n 55,\n 51,\n 55,\n 100,\n 101,\n 101,\n 52,\n 50,\n 51,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.701264546Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::6]:40796","req_id":"5433eb6c-a194-4ac0-b2f1-15f27c09e0f5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58126,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 53,\n 49,\n 53,\n 97,\n 101,\n 55,\n 45,\n 50,\n 49,\n 100,\n 49,\n 45,\n 52,\n 48,\n 101,\n 49,\n 45,\n 97,\n 100,\n 56,\n 55,\n 45,\n 55,\n 51,\n 55,\n 100,\n 101,\n 101,\n 52,\n 50,\n 51,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0515ae7-21d1-40e1-ad87-737dee42347d.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.702053052Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::6]:53295","req_id":"6a82abb6-4ba8-4efb-b92c-641a9e4a16e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21664,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.730441003Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:35607","req_id":"28484d92-9516-4147-919f-86d612cc15d5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21664,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 56,\n 56,\n 56,\n 97,\n 99,\n 45,\n 48,\n 97,\n 52,\n 55,\n 45,\n 52,\n 100,\n 55,\n 101,\n 45,\n 98,\n 97,\n 102,\n 98,\n 45,\n 53,\n 54,\n 101,\n 99,\n 101,\n 49,\n 49,\n 48,\n 102,\n 56,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_445888ac-0a47-4d7e-bafb-56ece110f8ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.73117799Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:54347","req_id":"017f416d-05ee-4e1e-b04d-08155b2ff89d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31393,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 55,\n 55,\n 51,\n 57,\n 56,\n 98,\n 45,\n 57,\n 101,\n 53,\n 97,\n 45,\n 52,\n 55,\n 56,\n 54,\n 45,\n 56,\n 53,\n 56,\n 52,\n 45,\n 55,\n 97,\n 55,\n 55,\n 49,\n 48,\n 97,\n 52,\n 101,\n 53,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.959139242Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::7]:37812","req_id":"63490959-4582-4eef-be28-9dd60290b199","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31393,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 55,\n 55,\n 51,\n 57,\n 56,\n 98,\n 45,\n 57,\n 101,\n 53,\n 97,\n 45,\n 52,\n 55,\n 56,\n 54,\n 45,\n 56,\n 53,\n 56,\n 52,\n 45,\n 55,\n 97,\n 55,\n 55,\n 49,\n 48,\n 97,\n 52,\n 101,\n 53,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d177398b-9e5a-4786-8584-7a7710a4e531.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.959872717Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::7]:34804","req_id":"8f84d572-3f1b-4138-8b8b-cecadb7174b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48284,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 100,\n 49,\n 50,\n 100,\n 49,\n 99,\n 51,\n 45,\n 102,\n 48,\n 49,\n 99,\n 45,\n 52,\n 53,\n 57,\n 55,\n 45,\n 98,\n 55,\n 57,\n 50,\n 45,\n 97,\n 48,\n 53,\n 55,\n 55,\n 56,\n 49,\n 100,\n 50,\n 100,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.966052341Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::3]:50483","req_id":"3f582a17-5d6b-4f5c-a7ee-374c4fe84761","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48284,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 100,\n 49,\n 50,\n 100,\n 49,\n 99,\n 51,\n 45,\n 102,\n 48,\n 49,\n 99,\n 45,\n 52,\n 53,\n 57,\n 55,\n 45,\n 98,\n 55,\n 57,\n 50,\n 45,\n 97,\n 48,\n 53,\n 55,\n 55,\n 56,\n 49,\n 100,\n 50,\n 100,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8d12d1c3-f01c-4597-b792-a057781d2d3a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-04T19:30:00.966742945Z","hostname":"oxz_internal_dns_c2077a3f-20f6-44b3-85fc-2b64bfb058d0","pid":5436,"peer_addr":"[fd00:1122:3344:101::3]:53840","req_id":"7429c9d5-e5d6-47cc-9e22-a534f5d19378","component":"dns"}