[ May 8 13:56:35 Enabled. ] [ May 8 13:56:35 Rereading configuration. ] [ May 8 13:56:36 Rereading configuration. ] [ May 8 13:56:37 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ May 8 13:56:37 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.622917488Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.623056671Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.642344747Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.642366117Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.642430548Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.642674873Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.774075233Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:34400"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.77495546Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"new_generation":1,"req_id":"4a20d579-1a50-4f01-b401-6e409d96e5fe","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.77498518Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.803507901Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.803541951Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"new_generation":1,"req_id":"4a20d579-1a50-4f01-b401-6e409d96e5fe","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:56:37.803553181Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"uri":"/config","method":"PUT","req_id":"4a20d579-1a50-4f01-b401-6e409d96e5fe","remote_addr":"[fd00:1122:3344:2::2]:34400","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":29092,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46645,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:56:48.807797919Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:48215","req_id":"febaf79d-f6f0-44a0-838f-dcb5e190ab65","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38074,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:56:48.812238613Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:36722","req_id":"4dc674ab-77d9-416a-b3b1-2e39fa0bbb9b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13050,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:56:48.816222468Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:37421","req_id":"55b59480-96c4-4e5c-80f0-dc5b8cf12025","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30685,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:58:49.508728143Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:36362","req_id":"f2079371-dcae-47eb-b10a-cf3f8cba47d9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54165,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:58:49.50949675Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:63964","req_id":"a07bd358-6da0-4d10-ad09-20b5bbae08f6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63262,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:58:49.50952765Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:42413","req_id":"7ae060f4-8bcd-484c-b576-182d94c3860e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20412,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:18.858048115Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:52007","req_id":"fd61eb9a-d35b-4b49-a559-f2f1c563a4fa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 242,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 55,\n 55,\n 57,\n 49,\n 100,\n 50,\n 45,\n 97,\n 98,\n 98,\n 49,\n 45,\n 52,\n 54,\n 49,\n 99,\n 45,\n 97,\n 99,\n 49,\n 50,\n 45,\n 49,\n 51,\n 49,\n 55,\n 52,\n 48,\n 53,\n 57,\n 55,\n 55,\n 55,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:22.17935453Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:49888","req_id":"264ff877-3b56-4059-ae43-83fcce5112a1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 242,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 55,\n 55,\n 57,\n 49,\n 100,\n 50,\n 45,\n 97,\n 98,\n 98,\n 49,\n 45,\n 52,\n 54,\n 49,\n 99,\n 45,\n 97,\n 99,\n 49,\n 50,\n 45,\n 49,\n 51,\n 49,\n 55,\n 52,\n 48,\n 53,\n 57,\n 55,\n 55,\n 55,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:22.180471021Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:37348","req_id":"f63f1b03-ff02-4b5f-9c19-b28211ce4139","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42908,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:27.222806815Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::a]:58472","req_id":"9eb1c6f2-f779-4bf3-8f73-875a03496b3a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21355,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:27.346778634Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::b]:50628","req_id":"3781a9fc-5463-4963-8a73-b65dd860ba1b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26693,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:27.347434346Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::c]:47119","req_id":"f7796e87-e326-48ce-81ee-12cb35624369","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21072,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:28.859709962Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:64269","req_id":"e274a7d6-cd3e-4665-93c5-b3ba4a282070","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57368,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:38.861603716Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:53858","req_id":"867b9155-57bd-4615-af11-3d8d43905643","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59017,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:48.862944879Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:54586","req_id":"11974830-fee2-47b7-8086-de6e0d5d4ce3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23789,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:57.416189941Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::c]:4166","req_id":"c1781e3b-2e58-4da8-8345-1a7564780d65","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:59:57.72276592Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:56789"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-08T13:59:57.798405353Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"uri":"/config","method":"GET","req_id":"ef70b9db-653f-4630-95e8-40ff118c1037","remote_addr":"[fd00:1122:3344:101::c]:56789","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":42787,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46493,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-08T13:59:58.864665807Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::d]:52758","req_id":"e708177a-16f0-430e-b6b8-d81d2518ab88","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18855,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 57,\n 49,\n 54,\n 52,\n 57,\n 52,\n 99,\n 45,\n 52,\n 102,\n 99,\n 101,\n 45,\n 52,\n 102,\n 50,\n 54,\n 45,\n 97,\n 53,\n 100,\n 50,\n 45,\n 101,\n 50,\n 53,\n 100,\n 99,\n 99,\n 51,\n 97,\n 56,\n 52,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.301548323Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::7]:62542","req_id":"ddd9dec5-3a39-4fb6-a47a-097e484b055d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18855,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 57,\n 49,\n 54,\n 52,\n 57,\n 52,\n 99,\n 45,\n 52,\n 102,\n 99,\n 101,\n 45,\n 52,\n 102,\n 50,\n 54,\n 45,\n 97,\n 53,\n 100,\n 50,\n 45,\n 101,\n 50,\n 53,\n 100,\n 99,\n 99,\n 51,\n 97,\n 56,\n 52,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0916494c-4fce-4f26-a5d2-e25dcc3a847e.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.302244726Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::7]:64145","req_id":"b7045f4a-315a-40cd-a50d-ae9c9a062048","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47486,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 48,\n 53,\n 98,\n 97,\n 102,\n 52,\n 54,\n 45,\n 49,\n 48,\n 55,\n 51,\n 45,\n 52,\n 50,\n 102,\n 98,\n 45,\n 57,\n 48,\n 48,\n 52,\n 45,\n 53,\n 53,\n 49,\n 50,\n 55,\n 52,\n 102,\n 55,\n 57,\n 55,\n 102,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.30510499Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::3]:54360","req_id":"697f1c00-d35a-422a-a5b7-efc8ff827339","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47486,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 48,\n 53,\n 98,\n 97,\n 102,\n 52,\n 54,\n 45,\n 49,\n 48,\n 55,\n 51,\n 45,\n 52,\n 50,\n 102,\n 98,\n 45,\n 57,\n 48,\n 48,\n 52,\n 45,\n 53,\n 53,\n 49,\n 50,\n 55,\n 52,\n 102,\n 55,\n 57,\n 55,\n 102,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_305baf46-1073-42fb-9004-551274f797fb.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.305579139Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::3]:39845","req_id":"74c09826-9b6a-4809-ba9f-b9a18366967a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28128,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 102,\n 99,\n 100,\n 55,\n 49,\n 53,\n 56,\n 45,\n 55,\n 56,\n 98,\n 54,\n 45,\n 52,\n 56,\n 56,\n 99,\n 45,\n 56,\n 52,\n 55,\n 55,\n 45,\n 52,\n 55,\n 56,\n 97,\n 100,\n 49,\n 48,\n 50,\n 56,\n 51,\n 55,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.323985228Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::5]:46366","req_id":"5a0a4752-5c1f-4774-a9a6-ca7c3bccc65d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28128,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 102,\n 99,\n 100,\n 55,\n 49,\n 53,\n 56,\n 45,\n 55,\n 56,\n 98,\n 54,\n 45,\n 52,\n 56,\n 56,\n 99,\n 45,\n 56,\n 52,\n 55,\n 55,\n 45,\n 52,\n 55,\n 56,\n 97,\n 100,\n 49,\n 48,\n 50,\n 56,\n 51,\n 55,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8fcd7158-78b6-488c-8477-478ad102837c.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.324503398Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::5]:40543","req_id":"f8eda5c7-5481-458a-befc-91737ad46526","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60125,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 48,\n 54,\n 49,\n 50,\n 99,\n 56,\n 100,\n 45,\n 99,\n 98,\n 51,\n 101,\n 45,\n 52,\n 97,\n 99,\n 49,\n 45,\n 56,\n 49,\n 98,\n 99,\n 45,\n 98,\n 57,\n 99,\n 56,\n 54,\n 56,\n 50,\n 52,\n 51,\n 51,\n 53,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.334778223Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::4]:49999","req_id":"c5049c80-05d7-45d0-9348-a5585afc77e2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60125,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 48,\n 54,\n 49,\n 50,\n 99,\n 56,\n 100,\n 45,\n 99,\n 98,\n 51,\n 101,\n 45,\n 52,\n 97,\n 99,\n 49,\n 45,\n 56,\n 49,\n 98,\n 99,\n 45,\n 98,\n 57,\n 99,\n 56,\n 54,\n 56,\n 50,\n 52,\n 51,\n 51,\n 53,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_20612c8d-cb3e-4ac1-81bc-b9c868243355.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.335376554Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::4]:45679","req_id":"3807842c-ee1e-4758-ad67-df1c27c1e231","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2092,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 102,\n 101,\n 55,\n 48,\n 55,\n 100,\n 49,\n 45,\n 53,\n 101,\n 51,\n 100,\n 45,\n 52,\n 50,\n 98,\n 51,\n 45,\n 98,\n 48,\n 51,\n 56,\n 45,\n 53,\n 55,\n 48,\n 54,\n 102,\n 55,\n 99,\n 52,\n 51,\n 54,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.338707147Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::6]:59612","req_id":"6586b74e-a18a-4486-8a75-e600704b8201","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2092,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 102,\n 101,\n 55,\n 48,\n 55,\n 100,\n 49,\n 45,\n 53,\n 101,\n 51,\n 100,\n 45,\n 52,\n 50,\n 98,\n 51,\n 45,\n 98,\n 48,\n 51,\n 56,\n 45,\n 53,\n 55,\n 48,\n 54,\n 102,\n 55,\n 99,\n 52,\n 51,\n 54,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5fe707d1-5e3d-42b3-b038-5706f7c436ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.339165766Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::6]:41983","req_id":"7701f486-04c2-447c-9c1b-736a435ce40d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16232,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.786312559Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::2]:59232","req_id":"039c387a-9b2f-47a1-97ec-854d708b7e6f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16232,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.786948771Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::2]:55341","req_id":"8eb2cc99-a161-4b92-aad5-66692276c3c5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50407,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 55,\n 55,\n 57,\n 49,\n 100,\n 50,\n 45,\n 97,\n 98,\n 98,\n 49,\n 45,\n 52,\n 54,\n 49,\n 99,\n 45,\n 97,\n 99,\n 49,\n 50,\n 45,\n 49,\n 51,\n 49,\n 55,\n 52,\n 48,\n 53,\n 57,\n 55,\n 55,\n 55,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.822398013Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:49859","req_id":"f36a0038-c5c8-4abd-b298-b6e7c1e92365","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50407,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 55,\n 55,\n 57,\n 49,\n 100,\n 50,\n 45,\n 97,\n 98,\n 98,\n 49,\n 45,\n 52,\n 54,\n 49,\n 99,\n 45,\n 97,\n 99,\n 49,\n 50,\n 45,\n 49,\n 51,\n 49,\n 55,\n 52,\n 48,\n 53,\n 57,\n 55,\n 55,\n 55,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7c7791d2-abb1-461c-ac12-131740597770.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.823061635Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::e]:48214","req_id":"c6588621-af80-4d48-ac8a-8b230f2261b8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65369,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_6b1d49a4-4cfc-482f-9b43-a852b5fc2148.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_6b1d49a4-4cfc-482f-9b43-a852b5fc2148.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 54,\n 98,\n 49,\n 100,\n 52,\n 57,\n 97,\n 52,\n 45,\n 52,\n 99,\n 102,\n 99,\n 45,\n 52,\n 56,\n 50,\n 102,\n 45,\n 57,\n 98,\n 52,\n 51,\n 45,\n 97,\n 56,\n 53,\n 50,\n 98,\n 53,\n 102,\n 99,\n 50,\n 49,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_6b1d49a4-4cfc-482f-9b43-a852b5fc2148.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-08T14:00:00.840392624Z","hostname":"oxz_internal_dns_14e2f5c1-1a66-490e-b5d7-09646b6fd25d","pid":5493,"peer_addr":"[fd00:1122:3344:101::1b]:35717","req_id":"3302e21c-67a6-42e3-8ea9-71fe4e13fd44","component":"dns"}