[ Jul 21 19:11:24 Enabled. ] [ Jul 21 19:11:24 Rereading configuration. ] [ Jul 21 19:11:25 Rereading configuration. ] [ Jul 21 19:11:26 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Jul 21 19:11:26 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.355286905Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.355477729Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.377905453Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.377926693Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.377966984Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.378285421Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12739,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:26.461451697Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::2]:41136","req_id":"3fa4cc14-e396-4035-82aa-898b1314e4d6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31251,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:26.642613584Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::2]:18810","req_id":"ce207634-d978-4a28-9dd8-aeb990bbc23e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64182,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:26.819400488Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::2]:5445","req_id":"ecc4f428-2c57-4f3a-b98f-0635f0d10141","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.868244989Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:56915"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.868841832Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"new_generation":1,"req_id":"d22449c6-283d-4061-98bd-db206c202df3","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.868859082Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.881146322Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.881178802Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"new_generation":1,"req_id":"d22449c6-283d-4061-98bd-db206c202df3","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:11:26.881213203Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"PUT","req_id":"d22449c6-283d-4061-98bd-db206c202df3","remote_addr":"[fd00:1122:3344:2::2]:56915","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":12683,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32338,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:37.659018492Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:61118","req_id":"b46bef53-4780-475c-8ba1-98f99f1a92de","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7495,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:37.663774483Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:52391","req_id":"145ca1af-7248-4e7f-b179-66208869ee08","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55999,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:11:37.669030464Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:61140","req_id":"cf4104cc-e612-40d1-9210-0631d013c2f8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34779,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:13:37.985802091Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:40273","req_id":"3b92b72b-513c-465c-9c72-777fc2de1c1f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4480,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:13:38.018531959Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:58503","req_id":"14c46053-99cc-4b3d-acc4-7b73f099cf0c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11169,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:13:38.033339814Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:33110","req_id":"dc618f09-193c-4dbf-b4da-c38f8052bf99","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17278,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:00.419842918Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:58174","req_id":"2f20afa3-c094-4b6a-b4c0-60a099101092","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13749,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 57,\n 98,\n 99,\n 100,\n 102,\n 45,\n 51,\n 55,\n 54,\n 55,\n 45,\n 52,\n 99,\n 99,\n 51,\n 45,\n 56,\n 55,\n 56,\n 48,\n 45,\n 50,\n 56,\n 51,\n 54,\n 101,\n 56,\n 55,\n 49,\n 100,\n 99,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:05.803302058Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::e]:48450","req_id":"23e0f052-3e33-4c88-a1fb-c56a1193e405","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13749,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 57,\n 98,\n 99,\n 100,\n 102,\n 45,\n 51,\n 55,\n 54,\n 55,\n 45,\n 52,\n 99,\n 99,\n 51,\n 45,\n 56,\n 55,\n 56,\n 48,\n 45,\n 50,\n 56,\n 51,\n 54,\n 101,\n 56,\n 55,\n 49,\n 100,\n 99,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:05.808187951Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::e]:42840","req_id":"dfa96375-8b25-40a2-bfec-0c243c73c8e9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37323,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:10.401750476Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:63157","req_id":"22dc6199-d50b-4bc9-b3f7-eb9abc1ea623","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44669,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:13.441426712Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::b]:10627","req_id":"3589a58a-4ddd-43c8-b24a-75616bd4afa0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20313,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:13.709644066Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::a]:64017","req_id":"e255cc18-940b-4783-8ab0-9e47470c8249","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11071,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:13.825167116Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::c]:43592","req_id":"f8d09762-ca87-4490-b069-468079b898df","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48167,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:20.40314083Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:49673","req_id":"70295063-cbdc-4352-a876-a1c2a6e60995","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31381,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:30.405070881Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:63651","req_id":"9d6b72b6-a124-412a-9a54-b9b44e48dbd4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41125,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:40.406139795Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:60226","req_id":"c142da29-12a9-4c1c-87cc-de501badb5b5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11147,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:46.109788174Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::c]:16568","req_id":"f5ab5d1f-949c-4b22-9869-4be420f0e398","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:46.424943109Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:53417"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:46.490960154Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"GET","req_id":"62a7c48e-9e03-437d-a880-70bc02ae80c1","remote_addr":"[fd00:1122:3344:101::c]:53417","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":35664,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15562,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:50.407302342Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:53096","req_id":"56f39673-e739-4564-9d80-ae8ec8a21d8f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49476,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:50.412050953Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::a]:37983","req_id":"24f37958-32ea-4021-840e-ecdc705e05fa","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.41949478Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45327"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.433800122Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"new_generation":1,"req_id":"1953409b-42bb-49c7-a15d-85bb4618b237","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.433829453Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"new_generation":1,"req_id":"1953409b-42bb-49c7-a15d-85bb4618b237","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.433890664Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"PUT","req_id":"1953409b-42bb-49c7-a15d-85bb4618b237","remote_addr":"[fd00:1122:3344:101::a]:45327","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":12826,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.540717271Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:62700"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:50.540982976Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"GET","req_id":"96296e49-3f42-41cf-8958-ddc6ea88701a","remote_addr":"[fd00:1122:3344:101::a]:62700","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":219,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23839,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:51.345673283Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::a]:1180","req_id":"b2380dca-0fed-426a-a010-1413b40d1a83","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:51.576968079Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:46963"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:51.577262025Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"GET","req_id":"d7ec80ef-dcf7-4ce6-8937-fe1aae10daac","remote_addr":"[fd00:1122:3344:101::a]:46963","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":240,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49790,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:54.194236104Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::b]:18063","req_id":"1b99d4e3-a4cc-4a85-9e5e-9383a09a10f9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32714,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:14:54.295940311Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::c]:29184","req_id":"56272b48-be47-43ad-aae4-33a66cb74c73","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:54.341772969Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38604"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:54.342020964Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"GET","req_id":"9b1ebbab-7e6a-408f-acbd-be93468f219b","remote_addr":"[fd00:1122:3344:101::b]:38604","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":196,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:54.419991661Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:58019"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-21T19:14:54.420272217Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"uri":"/config","method":"GET","req_id":"6f7a06f6-3148-4fc6-bf31-3e512025c25d","remote_addr":"[fd00:1122:3344:101::c]:58019","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":224,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27101,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.40806051Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::d]:55247","req_id":"328edc86-808a-4a7c-a1f4-38e9bf2b463d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55605,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.451226582Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::2]:44917","req_id":"3eacb9be-05f9-41da-bcfe-4e34a8069139","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55605,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.451823334Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::2]:40998","req_id":"3e6c5826-91a9-4fb9-a21f-5638e2ef2815","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 98,\n 54,\n 98,\n 48,\n 50,\n 50,\n 49,\n 45,\n 49,\n 97,\n 50,\n 99,\n 45,\n 52,\n 97,\n 48,\n 54,\n 45,\n 97,\n 54,\n 48,\n 53,\n 45,\n 55,\n 102,\n 50,\n 54,\n 49,\n 100,\n 49,\n 52,\n 51,\n 49,\n 99,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.854910329Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::7]:35596","req_id":"78e48539-aecd-4c7c-b98c-4424eb60daf2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 98,\n 54,\n 98,\n 48,\n 50,\n 50,\n 49,\n 45,\n 49,\n 97,\n 50,\n 99,\n 45,\n 52,\n 97,\n 48,\n 54,\n 45,\n 97,\n 54,\n 48,\n 53,\n 45,\n 55,\n 102,\n 50,\n 54,\n 49,\n 100,\n 49,\n 52,\n 51,\n 49,\n 99,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5b6b0221-1a2c-4a06-a605-7f261d1431c2.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.855674015Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::7]:53560","req_id":"575b6faa-2ca1-479e-a6a3-1ee2d40361e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42330,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 57,\n 98,\n 99,\n 100,\n 102,\n 45,\n 51,\n 55,\n 54,\n 55,\n 45,\n 52,\n 99,\n 99,\n 51,\n 45,\n 56,\n 55,\n 56,\n 48,\n 45,\n 50,\n 56,\n 51,\n 54,\n 101,\n 56,\n 55,\n 49,\n 100,\n 99,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.944093663Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::e]:64147","req_id":"52316513-3c1f-41dd-b180-8afce923e42e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42330,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 52,\n 52,\n 53,\n 57,\n 98,\n 99,\n 100,\n 102,\n 45,\n 51,\n 55,\n 54,\n 55,\n 45,\n 52,\n 99,\n 99,\n 51,\n 45,\n 56,\n 55,\n 56,\n 48,\n 45,\n 50,\n 56,\n 51,\n 54,\n 101,\n 56,\n 55,\n 49,\n 100,\n 99,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_4459bcdf-3767-4cc3-8780-2836e871dc57.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.944948431Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::e]:57962","req_id":"6e5c95bf-28e2-4481-b529-d0e3024b7407","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5672,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 99,\n 101,\n 50,\n 52,\n 54,\n 50,\n 50,\n 45,\n 48,\n 54,\n 100,\n 99,\n 45,\n 52,\n 50,\n 56,\n 57,\n 45,\n 98,\n 56,\n 101,\n 57,\n 45,\n 51,\n 48,\n 57,\n 57,\n 48,\n 48,\n 99,\n 48,\n 102,\n 51,\n 97,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.948257831Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::3]:34709","req_id":"1abd575c-3389-4f19-b612-0c6bc4bb3058","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5672,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 99,\n 101,\n 50,\n 52,\n 54,\n 50,\n 50,\n 45,\n 48,\n 54,\n 100,\n 99,\n 45,\n 52,\n 50,\n 56,\n 57,\n 45,\n 98,\n 56,\n 101,\n 57,\n 45,\n 51,\n 48,\n 57,\n 57,\n 48,\n 48,\n 99,\n 48,\n 102,\n 51,\n 97,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9ce24622-06dc-4289-b8e9-309900c0f3a6.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.948976126Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::3]:45049","req_id":"a6442962-bcde-4530-a30f-b8e0f5d18202","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54274,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 55,\n 54,\n 50,\n 53,\n 100,\n 53,\n 45,\n 48,\n 101,\n 52,\n 53,\n 45,\n 52,\n 97,\n 53,\n 53,\n 45,\n 57,\n 101,\n 49,\n 99,\n 45,\n 99,\n 102,\n 49,\n 51,\n 54,\n 57,\n 54,\n 52,\n 100,\n 56,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.955242748Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::5]:61895","req_id":"404ea062-8b7a-4e19-991d-ce91664c1508","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54274,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 55,\n 54,\n 50,\n 53,\n 100,\n 53,\n 45,\n 48,\n 101,\n 52,\n 53,\n 45,\n 52,\n 97,\n 53,\n 53,\n 45,\n 57,\n 101,\n 49,\n 99,\n 45,\n 99,\n 102,\n 49,\n 51,\n 54,\n 57,\n 54,\n 52,\n 100,\n 56,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be7625d5-0e45-4a55-9e1c-cf136964d87e.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.955933073Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::5]:50654","req_id":"017d9d01-fc21-4a49-8aaf-52f9065ba04c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 56,\n 97,\n 102,\n 55,\n 51,\n 99,\n 45,\n 48,\n 100,\n 52,\n 100,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 48,\n 99,\n 56,\n 45,\n 97,\n 53,\n 56,\n 56,\n 51,\n 99,\n 100,\n 100,\n 99,\n 57,\n 57,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.980323198Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::6]:51439","req_id":"79feafd9-048f-472e-abc9-c1537892360c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 56,\n 97,\n 102,\n 55,\n 51,\n 99,\n 45,\n 48,\n 100,\n 52,\n 100,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 48,\n 99,\n 56,\n 45,\n 97,\n 53,\n 56,\n 56,\n 51,\n 99,\n 100,\n 100,\n 99,\n 57,\n 57,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be8af73c-0d4d-4de4-90c8-a5883cddc992.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.981201936Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::6]:42740","req_id":"3d492468-3838-4597-ac59-138b9a22dc68","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18722,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 52,\n 54,\n 52,\n 49,\n 98,\n 51,\n 45,\n 97,\n 54,\n 99,\n 57,\n 45,\n 52,\n 54,\n 50,\n 97,\n 45,\n 56,\n 53,\n 54,\n 53,\n 45,\n 101,\n 50,\n 50,\n 48,\n 50,\n 56,\n 48,\n 101,\n 54,\n 50,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.981704357Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::4]:46185","req_id":"ad3c9043-ada7-4e0a-9ea1-7078a1b98584","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18722,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 101,\n 52,\n 54,\n 52,\n 49,\n 98,\n 51,\n 45,\n 97,\n 54,\n 99,\n 57,\n 45,\n 52,\n 54,\n 50,\n 97,\n 45,\n 56,\n 53,\n 54,\n 53,\n 45,\n 101,\n 50,\n 50,\n 48,\n 50,\n 56,\n 48,\n 101,\n 54,\n 50,\n 51,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_be4641b3-a6c9-462a-8565-e220280e6231.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:00.982454023Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::4]:40909","req_id":"b6f7d168-6eb6-4bdb-a76f-0918c185f8a7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7509,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_812005db-4218-4f5c-b6fb-6fe4ef68eb12.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_812005db-4218-4f5c-b6fb-6fe4ef68eb12.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 56,\n 49,\n 50,\n 48,\n 48,\n 53,\n 100,\n 98,\n 45,\n 52,\n 50,\n 49,\n 56,\n 45,\n 52,\n 102,\n 53,\n 99,\n 45,\n 98,\n 54,\n 102,\n 98,\n 45,\n 54,\n 102,\n 101,\n 52,\n 101,\n 102,\n 54,\n 56,\n 101,\n 98,\n 49,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_812005db-4218-4f5c-b6fb-6fe4ef68eb12.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-21T19:15:01.208338604Z","hostname":"oxz_internal_dns_79d29063-0ac0-4341-8cf9-a14317cd58f6","pid":5422,"peer_addr":"[fd00:1122:3344:101::1b]:34512","req_id":"684a2252-9932-4c32-91ad-9cb760483be3","component":"dns"}