[ Apr 17 13:57:47 Enabled. ] [ Apr 17 13:57:47 Rereading configuration. ] [ Apr 17 13:57:48 Rereading configuration. ] [ Apr 17 13:57:49 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Apr 17 13:57:49 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.51032054Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.510448182Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.530400069Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.530412929Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.53047429Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.530668762Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.908258479Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:57511"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.909175959Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"new_generation":1,"req_id":"937f0eea-4951-4da6-8adb-b7e0d5fb4072","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.909191349Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.92591188Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.92594232Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"new_generation":1,"req_id":"937f0eea-4951-4da6-8adb-b7e0d5fb4072","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.92595682Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"uri":"/config","method":"PUT","req_id":"937f0eea-4951-4da6-8adb-b7e0d5fb4072","remote_addr":"[fd00:1122:3344:3::2]:57511","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":17201,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18268,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.928486896Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:47344","req_id":"92fdd517-9ac3-4a07-b65f-851f716ae22e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52065,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.933042438Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:34484","req_id":"dbdb73db-23cd-4237-a1e3-00192a2058ed","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27052,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.937132855Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:56814","req_id":"91411fc9-8d53-48d9-85f3-8c1f4c83f7a7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20783,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:54.7210201Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:54980","req_id":"e634a96c-9b53-4cff-95c7-b1e27f9d2929","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36872,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:58.051982869Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:52607","req_id":"40f0ae43-a603-4909-8dd1-331d9dd55431","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36872,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:58.052755208Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:52129","req_id":"1778e398-abfe-4251-a92b-f9148f662abc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64315,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 57,\n 55,\n 49,\n 51,\n 49,\n 52,\n 101,\n 45,\n 54,\n 100,\n 49,\n 52,\n 45,\n 52,\n 101,\n 102,\n 57,\n 45,\n 56,\n 101,\n 53,\n 98,\n 45,\n 51,\n 56,\n 102,\n 100,\n 55,\n 102,\n 101,\n 102,\n 52,\n 98,\n 101,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.274742329Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::6]:59357","req_id":"65a26cd2-f3fc-40cb-83b3-71a77f655ed6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64315,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 57,\n 55,\n 49,\n 51,\n 49,\n 52,\n 101,\n 45,\n 54,\n 100,\n 49,\n 52,\n 45,\n 52,\n 101,\n 102,\n 57,\n 45,\n 56,\n 101,\n 53,\n 98,\n 45,\n 51,\n 56,\n 102,\n 100,\n 55,\n 102,\n 101,\n 102,\n 52,\n 98,\n 101,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.275957633Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::6]:62310","req_id":"93076dcd-000b-407c-bfdb-99ae18e7bd63","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 54,\n 100,\n 50,\n 52,\n 97,\n 99,\n 45,\n 49,\n 101,\n 48,\n 57,\n 45,\n 52,\n 51,\n 52,\n 51,\n 45,\n 97,\n 57,\n 102,\n 102,\n 45,\n 100,\n 100,\n 48,\n 48,\n 101,\n 55,\n 100,\n 48,\n 51,\n 57,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.450518401Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::7]:58268","req_id":"48c92d7c-99e4-4a41-a7cf-2ffffb2a9e80","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 54,\n 100,\n 50,\n 52,\n 97,\n 99,\n 45,\n 49,\n 101,\n 48,\n 57,\n 45,\n 52,\n 51,\n 52,\n 51,\n 45,\n 97,\n 57,\n 102,\n 102,\n 45,\n 100,\n 100,\n 48,\n 48,\n 101,\n 55,\n 100,\n 48,\n 51,\n 57,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.451150858Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::7]:37299","req_id":"477d575d-bd05-471e-b62b-f7c6d92d80e1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 56,\n 49,\n 101,\n 99,\n 97,\n 102,\n 45,\n 52,\n 98,\n 99,\n 48,\n 45,\n 52,\n 100,\n 57,\n 99,\n 45,\n 97,\n 49,\n 49,\n 98,\n 45,\n 97,\n 100,\n 97,\n 56,\n 54,\n 101,\n 48,\n 101,\n 102,\n 52,\n 98,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.538447522Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::5]:49984","req_id":"558e83c4-bbfe-4311-a63f-b02da3146d48","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 56,\n 49,\n 101,\n 99,\n 97,\n 102,\n 45,\n 52,\n 98,\n 99,\n 48,\n 45,\n 52,\n 100,\n 57,\n 99,\n 45,\n 97,\n 49,\n 49,\n 98,\n 45,\n 97,\n 100,\n 97,\n 56,\n 54,\n 101,\n 48,\n 101,\n 102,\n 52,\n 98,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.539227321Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::5]:50184","req_id":"c25668b3-61ce-43f1-88f5-3649bf830468","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 98,\n 55,\n 52,\n 102,\n 100,\n 48,\n 45,\n 48,\n 54,\n 55,\n 97,\n 45,\n 52,\n 54,\n 99,\n 100,\n 45,\n 97,\n 55,\n 97,\n 99,\n 45,\n 99,\n 53,\n 51,\n 49,\n 100,\n 102,\n 49,\n 102,\n 55,\n 100,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.543449159Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::3]:61046","req_id":"89c535d1-5008-4cba-9b95-1aa991836a54","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 98,\n 55,\n 52,\n 102,\n 100,\n 48,\n 45,\n 48,\n 54,\n 55,\n 97,\n 45,\n 52,\n 54,\n 99,\n 100,\n 45,\n 97,\n 55,\n 97,\n 99,\n 45,\n 99,\n 53,\n 51,\n 49,\n 100,\n 102,\n 49,\n 102,\n 55,\n 100,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.544090666Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::3]:49080","req_id":"b147914a-05c9-4fb7-a583-911105b57518","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35542,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 54,\n 100,\n 99,\n 55,\n 100,\n 55,\n 100,\n 45,\n 98,\n 56,\n 102,\n 54,\n 45,\n 52,\n 56,\n 53,\n 53,\n 45,\n 97,\n 55,\n 98,\n 57,\n 45,\n 54,\n 48,\n 52,\n 56,\n 49,\n 98,\n 51,\n 99,\n 51,\n 55,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.562799709Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::4]:60797","req_id":"dd7d1945-353d-4834-a310-e251f9fa1eda","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35542,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 54,\n 100,\n 99,\n 55,\n 100,\n 55,\n 100,\n 45,\n 98,\n 56,\n 102,\n 54,\n 45,\n 52,\n 56,\n 53,\n 53,\n 45,\n 97,\n 55,\n 98,\n 57,\n 45,\n 54,\n 48,\n 52,\n 56,\n 49,\n 98,\n 51,\n 99,\n 51,\n 55,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.563496007Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::4]:49287","req_id":"e30c9ee3-7d56-4b8e-a452-39b96636cbff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 124,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 48,\n 101,\n 100,\n 55,\n 53,\n 55,\n 53,\n 53,\n 45,\n 55,\n 102,\n 52,\n 57,\n 45,\n 52,\n 48,\n 54,\n 55,\n 45,\n 56,\n 57,\n 55,\n 54,\n 45,\n 53,\n 99,\n 99,\n 53,\n 50,\n 98,\n 55,\n 51,\n 54,\n 57,\n 52,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.666079715Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:33987","req_id":"681ac212-dacf-4f1a-8021-27212d131381","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2636,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.703695824Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:47110","req_id":"2b66bb4a-6b9c-4f5d-ae9c-db6f2ed94778","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2636,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.704526253Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:62737","req_id":"ccc83999-0cd5-4586-907a-091b7ad97000","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34012,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:01.410917017Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::2]:48131","req_id":"bcfa1c6d-d4ab-431f-9e52-4069e0447a16","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34012,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:01.411488794Z","hostname":"oxz_internal_dns_60541528-8156-4e75-9311-d0e8d98a0b83","pid":5493,"peer_addr":"[fd00:1122:3344:101::2]:53386","req_id":"937fb10c-d899-4a53-9193-bbe40efdee8b","component":"dns"}