[ Mar 24 14:56:42 Enabled. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ Mar 24 14:56:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.31473105Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.314882003Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.338086427Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.338101357Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.338146489Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.338476516Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.08225873Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:35655"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.083017867Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"857e251a-a6b7-432e-8634-ddf3ecae7851","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.083039938Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.11271597Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.112748091Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"857e251a-a6b7-432e-8634-ddf3ecae7851","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.112764571Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"PUT","req_id":"857e251a-a6b7-432e-8634-ddf3ecae7851","remote_addr":"[fd00:1122:3344:1::2]:35655","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":30127,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54632,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.149548733Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:64817","req_id":"71b199c6-97c7-4554-b4df-9263111347aa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14193,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.153910574Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:40185","req_id":"edd1a525-18b4-460f-ad46-a5f7dede936d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.158278054Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:46452","req_id":"5874a346-2531-4a6c-91c6-b0d9385cb39f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53781,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:20.172241207Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:58632","req_id":"a151c880-bbba-47b3-aa49-1cb1119ceaf0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.247987963Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:49628","req_id":"952a68b9-8bb0-41de-9cdc-7b0d154acc90","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.266948749Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:33142","req_id":"8c377c98-8a9c-4d6d-846c-8ea9f67aa5ba","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9948,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:30.131527029Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:62892","req_id":"f9732ffb-8e34-4eec-9a70-94b035b3a6de","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36112,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:32.606288199Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:6960","req_id":"d730102b-6526-4f8a-9714-54d75b3f5dd6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39583,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.732826805Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:7293","req_id":"35a751cf-0265-40e3-bbde-c54d2ce362e1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20246,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.841093034Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:44150","req_id":"518cf6a8-51bb-4f54-9539-415c2a4e259b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41179,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:40.13248096Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:63739","req_id":"f9e1bbb3-cc12-4842-93c5-9d060cc9ecdb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45118,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:50.133623024Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:52032","req_id":"ea6e3b6b-838a-4bc5-bee8-677c10e07e73","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46552,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.141720805Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:59914","req_id":"b04c8dd0-4ed3-469b-8c2a-864d8cf8948c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.146001434Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:34717","req_id":"e787ee6f-a741-45d2-b35c-c021af24db0c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26822,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.150316063Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:63436","req_id":"b767f236-be1b-479f-bde0-9636c391c92a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24113,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:00.135134697Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:56674","req_id":"06f49df9-3fc4-4565-a362-7186d4c0ffc2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1836,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:03.874908578Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:35771","req_id":"4f2ee34d-b417-461a-9fbb-98d3571a65d4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.133683958Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63900"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.190661867Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"d502e3a9-5ea2-416d-84c3-329a17fe7211","remote_addr":"[fd00:1122:3344:101::a]:63900","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":27670,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.739640574Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:53901"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741874717Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"41c4cccb-20c8-48bf-9547-e5d9ba87ee2e","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741935909Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"41c4cccb-20c8-48bf-9547-e5d9ba87ee2e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741972117Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"PUT","req_id":"41c4cccb-20c8-48bf-9547-e5d9ba87ee2e","remote_addr":"[fd00:1122:3344:101::a]:53901","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":307,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41274,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:05.073504224Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:28262","req_id":"b324ffea-9e91-4753-93dd-4a8d870b6283","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.36381697Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:38476"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.364100536Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"add2cf72-fac1-436c-9308-43a8305ce210","remote_addr":"[fd00:1122:3344:101::a]:38476","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":230,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56886,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:06.431501166Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:60532","req_id":"044023f9-5cb7-4e52-8bec-1faafc0beb8a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.55281445Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:55136"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.553118057Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"9dbf634e-3d53-4402-b67d-27fdd51391a8","remote_addr":"[fd00:1122:3344:101::a]:55136","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":277,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7372,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:10.138065817Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:60681","req_id":"5b9f747b-410f-44ac-9b90-08fd75ac94ab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14333,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:13.446134397Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:58605","req_id":"020f39ec-f048-4d67-bfd8-9df438c0676a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.58756185Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:34353"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.587871777Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"890f309a-50e7-4b1c-b4c5-567a01970ba0","remote_addr":"[fd00:1122:3344:101::b]:34353","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":232,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58367,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:14.4049192Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:52415","req_id":"6e71a89e-6ce0-4de2-b1ad-a22ad02d89b2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.524584871Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:39095"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.524839317Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"bc0a48fb-9855-45cd-b512-51c306180c2d","remote_addr":"[fd00:1122:3344:101::c]:39095","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":200,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38041,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:20.13791072Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:61075","req_id":"029f1b2f-14cc-4643-929b-6f0ae1893964","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17293,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:30.138824181Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:60589","req_id":"1d680ac2-1a2d-4c2b-b664-ee3728e708c0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.510823678Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:60063"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513930379Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"e5af427d-42c1-4f6c-bb60-935d010d2ac6","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513984291Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"e5af427d-42c1-4f6c-bb60-935d010d2ac6","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513996931Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"PUT","req_id":"e5af427d-42c1-4f6c-bb60-935d010d2ac6","remote_addr":"[fd00:1122:3344:101::b]:60063","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":221,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.621048479Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:57480"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.623921816Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"309a7133-c736-48f1-83b7-3094bb42846d","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.623990647Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"309a7133-c736-48f1-83b7-3094bb42846d","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.624039388Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"PUT","req_id":"309a7133-c736-48f1-83b7-3094bb42846d","remote_addr":"[fd00:1122:3344:101::a]:57480","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":252,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.79063647Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:64566"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794846696Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"7650437f-2bd6-48aa-b586-13ed84548644","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794874017Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"new_generation":1,"req_id":"7650437f-2bd6-48aa-b586-13ed84548644","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794906828Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"PUT","req_id":"7650437f-2bd6-48aa-b586-13ed84548644","remote_addr":"[fd00:1122:3344:101::c]:64566","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":1030,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2440,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:33.976707979Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:8871","req_id":"2b9fab74-a1f7-46e2-a19c-3bce758841fc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.211302145Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:57943"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.211569041Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"39ee175c-e4e1-4366-8ab8-dc7073412afa","remote_addr":"[fd00:1122:3344:101::b]:57943","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":220,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28646,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:34.623093458Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:30778","req_id":"6386151b-9173-4593-87f3-86995fb3eee6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.716567208Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:41159"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.716844034Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"b0f14204-08cc-4ce7-bb3d-9bd21fb747b4","remote_addr":"[fd00:1122:3344:101::c]:41159","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":222,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5872,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:35.161677627Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:4811","req_id":"11fd6869-e117-41a2-abf9-e30c95da71b6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.253870678Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:61645"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.254112224Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"uri":"/config","method":"GET","req_id":"4577c261-cfbd-4587-a196-d4f3532e0aa4","remote_addr":"[fd00:1122:3344:101::a]:61645","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45654,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:40.139499711Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:59973","req_id":"f40c7ca1-3125-4737-aa3d-d9cd4ce7b85d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1544,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:50.140800002Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:50453","req_id":"efba4882-0ae7-4b13-bbed-05040ea6ac53","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30279,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.142652044Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:59366","req_id":"e86f6883-913b-4eff-bcd7-3512d628e246","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.251688621Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::7]:33724","req_id":"f6440c1a-9cb4-432c-a7eb-a427c03d628a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.252263015Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::7]:38387","req_id":"d6a4fed4-489f-493d-9f34-8db7e232fd36","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.299838749Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::4]:35737","req_id":"80a589ec-9dec-44af-ba75-c04c4a8ad628","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.300483323Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::4]:59241","req_id":"9fa20f1e-fe8c-4508-acd8-c6d61c651f35","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.405838016Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::3]:50975","req_id":"ca4c8acd-c906-48c8-b614-2030ff288360","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.406609974Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::3]:62899","req_id":"1f99968f-8f5f-4e1a-adba-66dc862dd3c5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.489369587Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:55195","req_id":"d93ed612-2079-4c38-b489-4c75d266f39b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.489973801Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:53959","req_id":"f83a780a-3801-4ed2-b976-4b3b11404ff3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.625281332Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::6]:44892","req_id":"4957d313-8ef0-48a1-b927-9682b030b23c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.62605318Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::6]:55570","req_id":"dad9c6e0-248d-4191-810a-7c2acf63b09b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.637241038Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::5]:44629","req_id":"3aa7bd79-f71c-4f81-b015-2a42040ffa0c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.637902723Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::5]:51445","req_id":"8cc1b1cc-5d56-4ee9-a4e2-251f3aa7dbc1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5721,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 102,\n 51,\n 55,\n 97,\n 52,\n 55,\n 100,\n 99,\n 45,\n 54,\n 53,\n 50,\n 54,\n 45,\n 52,\n 57,\n 57,\n 53,\n 45,\n 57,\n 54,\n 54,\n 57,\n 45,\n 57,\n 97,\n 101,\n 98,\n 51,\n 102,\n 98,\n 54,\n 48,\n 54,\n 100,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.665325673Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:52321","req_id":"c37d40a2-cc55-466f-a688-e18a1a4aa146","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.857224196Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::2]:62492","req_id":"2a6b8ea8-17aa-42c6-adc2-ff607adbc79d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.857852341Z","hostname":"oxz_internal_dns_0cbfbf8c-b2c2-40e7-985f-4c790cfe4e6d","pid":5497,"peer_addr":"[fd00:1122:3344:101::2]:33790","req_id":"db45a1e0-611d-45c5-9399-d94ec3acc05d","component":"dns"}