[ Mar 24 14:56:42 Enabled. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:42 Rereading configuration. ] [ Mar 24 14:56:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Mar 24 14:56:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.322201172Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.322365146Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.343269596Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.343288167Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.343333848Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:44.343625495Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.11446161Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:42771"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.115257409Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"cae22650-6464-4eea-8fbb-51ee6ccb7295","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.115270469Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.145873293Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.145911324Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"cae22650-6464-4eea-8fbb-51ee6ccb7295","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:56:45.145941144Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"PUT","req_id":"cae22650-6464-4eea-8fbb-51ee6ccb7295","remote_addr":"[fd00:1122:3344:2::2]:42771","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":31053,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54632,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.149874861Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:41280","req_id":"d442f227-2b2b-4969-805f-fd581dc0cbee","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14193,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.154090488Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:49437","req_id":"078005ee-ef79-40cd-827b-3ce3656228e9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:56:56.158390557Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:52024","req_id":"772ea23c-b3ea-47ee-87dd-266f4738f63f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12888,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:20.175783929Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:52693","req_id":"4fbffe2f-de39-4f01-907e-3993a6c4b9b6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.258728699Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:34045","req_id":"aecc84aa-b706-4a9e-809b-0678349c96a3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:24.267141673Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:47405","req_id":"6c2beb85-13f3-44dd-b651-71f4d75995a6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43589,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:30.131529889Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:64467","req_id":"c54e2ff9-66be-4940-8926-a8cf67d343b3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15627,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:32.606230227Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:5163","req_id":"3d96cbeb-6c79-417b-afbd-a07109b3bc86","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3787,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.732801004Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:22473","req_id":"982f7508-c2f4-49d8-a465-a25de0732ec6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44475,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:33.841085894Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:53950","req_id":"8937a384-87f9-4fb0-8b50-2ce35f0f87b1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14677,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:40.13249124Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:49594","req_id":"8b95b0c5-b4a4-4c63-95ca-41eeb5969322","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9633,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:50.133673685Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:56042","req_id":"c2165ac4-6d21-4458-a2bb-777a33fd1b36","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46552,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.142059383Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:43156","req_id":"e80e40b5-cb55-423a-a5ed-82b047d94ede","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.146195458Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:43045","req_id":"616b0a67-fdc7-42f7-895f-63a375eb601f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26822,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:58:56.150567539Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:63641","req_id":"9a7d1adc-f76e-4d3d-b21f-a5830e3fd39a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5640,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:00.135154358Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:60834","req_id":"ee9075f0-24a7-40ae-b5b4-d2c66a2bffe8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23454,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:03.874913153Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:57365","req_id":"6a900844-bb07-4c6a-bdd4-0e2b4832e28c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.203088243Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:42175"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.268966117Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"6b651fa2-f40d-4464-8275-3d05af98771b","remote_addr":"[fd00:1122:3344:101::a]:42175","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":37280,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.740524422Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33439"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741905196Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"39454d2b-44cf-4092-b1ff-a831a957e2e0","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.741932169Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"39454d2b-44cf-4092-b1ff-a831a957e2e0","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:04.74203793Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"PUT","req_id":"39454d2b-44cf-4092-b1ff-a831a957e2e0","remote_addr":"[fd00:1122:3344:101::a]:33439","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":363,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36994,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:05.073470821Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:52138","req_id":"ee573b53-5c3e-4360-9be5-783c4116990d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.364520387Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:64353"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:05.364816653Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"3bf33f2b-bccd-4fd5-94f6-7970392f6e0d","remote_addr":"[fd00:1122:3344:101::a]:64353","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":239,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14864,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:06.431399178Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:62837","req_id":"e866cc8a-e8d3-44e9-944c-dd02502e74b4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.553571497Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33700"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:06.553878254Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"af7ad402-edc9-45cc-a2de-aa7889878881","remote_addr":"[fd00:1122:3344:101::a]:33700","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":231,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39039,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:10.138067257Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:57137","req_id":"f0b3adae-7272-400e-9447-fd7de8fe59e8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24701,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:13.446158498Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:24885","req_id":"ee5665f1-ae2c-4fcd-85da-a4adaa7fc465","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.588384399Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:36511"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:13.588659476Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"462a1133-b11e-4bf1-8893-4b371f3d87f9","remote_addr":"[fd00:1122:3344:101::b]:36511","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":235,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48026,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:14.404917369Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:42667","req_id":"906167cf-c70e-40bf-bcfb-44b4e7126d8b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.525365409Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:46836"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:14.526117056Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"448aec2c-9694-4af3-8d5f-cc6a3aa6c614","remote_addr":"[fd00:1122:3344:101::c]:46836","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":206,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31405,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:20.137828539Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:51819","req_id":"cea042a3-cac7-4246-b1e6-5a4b0c5ccab8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32525,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:30.138821381Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:58816","req_id":"d72fc215-ab97-4661-8ff3-79fe4ddb4f64","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.512500157Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:44713"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513930059Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"3bb0fd6e-1310-4dd6-90e7-f024892f965e","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513981421Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"3bb0fd6e-1310-4dd6-90e7-f024892f965e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:32.513991831Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"PUT","req_id":"3bb0fd6e-1310-4dd6-90e7-f024892f965e","remote_addr":"[fd00:1122:3344:101::b]:44713","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":232,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.622231186Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:65082"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.62412494Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"9b534d0a-f57f-4167-908f-e0001f55833a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.624191552Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"9b534d0a-f57f-4167-908f-e0001f55833a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.624209992Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"PUT","req_id":"9b534d0a-f57f-4167-908f-e0001f55833a","remote_addr":"[fd00:1122:3344:101::a]:65082","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":479,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.792332929Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:51857"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794041598Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"d59632ae-0c4e-471c-9172-3a0be1292207","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.794070698Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"new_generation":1,"req_id":"d59632ae-0c4e-471c-9172-3a0be1292207","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:33.79412037Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"PUT","req_id":"d59632ae-0c4e-471c-9172-3a0be1292207","remote_addr":"[fd00:1122:3344:101::c]:51857","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":252,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43033,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:33.976699769Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:16096","req_id":"21db012b-8a07-4098-9ede-d62d4a196a41","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.212077743Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50119"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.21237475Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"d73736ed-9b8c-4a2b-9a26-a00505fcc4cd","remote_addr":"[fd00:1122:3344:101::b]:50119","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":244,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44339,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:34.623105848Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:23421","req_id":"f7e9d86a-dbe5-4d6e-9d33-90ecd8eadbd6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.717306825Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:56352"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:34.717590531Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"14f8aa4d-7a6e-4ecf-b07f-a70e9dcc3f99","remote_addr":"[fd00:1122:3344:101::c]:56352","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":231,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 448,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:35.161683007Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:18942","req_id":"468b2495-1735-4b43-9d60-fae696ebe358","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.254603185Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:64879"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T14:59:35.254855301Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"uri":"/config","method":"GET","req_id":"89cbcf65-3dcc-45ae-91fc-406885732036","remote_addr":"[fd00:1122:3344:101::a]:64879","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":203,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63398,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:40.139483901Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:57132","req_id":"8491f63b-7afa-4db8-886e-7dafacfcf9cb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7044,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T14:59:50.140797862Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:50342","req_id":"12b5991d-2b3a-4176-bda4-3444bd34eb99","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64991,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.142631273Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:54392","req_id":"0d9d0fdb-9a17-4f24-980b-81beeb8bee92","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.251951018Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::7]:43936","req_id":"ce72910e-67fa-4f2c-ac2a-913be4ca1f28","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42219,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 54,\n 53,\n 99,\n 98,\n 54,\n 102,\n 53,\n 45,\n 49,\n 57,\n 51,\n 56,\n 45,\n 52,\n 54,\n 98,\n 57,\n 45,\n 57,\n 101,\n 98,\n 54,\n 45,\n 97,\n 102,\n 98,\n 53,\n 48,\n 52,\n 102,\n 101,\n 49,\n 54,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b65cb6f5-1938-46b9-9eb6-afb504fe163d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.252459809Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::7]:46462","req_id":"c5b44665-3bb5-4f72-a157-a3978b8ebbdd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.300113085Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::4]:44809","req_id":"6f14407b-a43b-4b85-86d9-f8b5f1b4ffca","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 99,\n 101,\n 99,\n 49,\n 49,\n 49,\n 45,\n 97,\n 48,\n 101,\n 53,\n 45,\n 52,\n 54,\n 49,\n 52,\n 45,\n 98,\n 51,\n 50,\n 49,\n 45,\n 99,\n 100,\n 57,\n 57,\n 99,\n 56,\n 57,\n 48,\n 99,\n 50,\n 101,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6bcec111-a0e5-4614-b321-cd99c890c2ed.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.300684898Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::4]:51632","req_id":"b699d0bb-c689-4ae9-be21-64eda477c8af","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.406178964Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::3]:45051","req_id":"2f8dd058-be74-4898-8ed2-c73cd61ab2a8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 55,\n 102,\n 57,\n 53,\n 101,\n 53,\n 54,\n 45,\n 98,\n 101,\n 97,\n 102,\n 45,\n 52,\n 55,\n 49,\n 52,\n 45,\n 98,\n 102,\n 48,\n 56,\n 45,\n 52,\n 99,\n 51,\n 49,\n 57,\n 49,\n 54,\n 98,\n 53,\n 55,\n 49,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_87f95e56-beaf-4714-bf08-4c31916b5719.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.406820549Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::3]:55239","req_id":"bd910566-8633-4bab-8fe6-0fe46b37911b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.489603492Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:34711","req_id":"470732a0-f279-4525-bd04-5edc212bafc3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53206,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 98,\n 51,\n 98,\n 55,\n 55,\n 51,\n 48,\n 100,\n 45,\n 101,\n 101,\n 53,\n 98,\n 45,\n 52,\n 102,\n 98,\n 99,\n 45,\n 98,\n 51,\n 57,\n 51,\n 45,\n 98,\n 51,\n 48,\n 49,\n 50,\n 100,\n 99,\n 55,\n 57,\n 97,\n 53,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_b3b7730d-ee5b-4fbc-b393-b3012dc79a51.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.490183216Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:55250","req_id":"f6c963b8-e8bd-4228-b244-af77d52c67e3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.62560301Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::6]:47810","req_id":"fc60be9c-4701-46cf-b6af-6b6a8b099470","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 98,\n 56,\n 56,\n 51,\n 52,\n 51,\n 45,\n 102,\n 53,\n 57,\n 51,\n 45,\n 52,\n 50,\n 98,\n 53,\n 45,\n 98,\n 49,\n 100,\n 56,\n 45,\n 98,\n 57,\n 56,\n 56,\n 100,\n 51,\n 100,\n 54,\n 101,\n 97,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65b88343-f593-42b5-b1d8-b988d3d6ea48.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.626315736Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::6]:35418","req_id":"8b8060c5-71b9-4292-99c5-c396c9db7c07","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.637505824Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::5]:50862","req_id":"648bda4c-eeb4-4e8f-9b1e-8483f8ca7111","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48051,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 49,\n 99,\n 51,\n 53,\n 48,\n 98,\n 52,\n 45,\n 99,\n 101,\n 51,\n 102,\n 45,\n 52,\n 57,\n 100,\n 101,\n 45,\n 98,\n 51,\n 55,\n 55,\n 45,\n 54,\n 50,\n 53,\n 101,\n 51,\n 99,\n 101,\n 54,\n 97,\n 100,\n 53,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f1c350b4-ce3f-49de-b377-625e3ce6ad5f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.638115487Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::5]:59765","req_id":"0f5f28a3-6994-4e0e-858b-31c56cc6c27e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5721,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 102,\n 51,\n 55,\n 97,\n 52,\n 55,\n 100,\n 99,\n 45,\n 54,\n 53,\n 50,\n 54,\n 45,\n 52,\n 57,\n 57,\n 53,\n 45,\n 57,\n 54,\n 54,\n 57,\n 45,\n 57,\n 97,\n 101,\n 98,\n 51,\n 102,\n 98,\n 54,\n 48,\n 54,\n 100,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_f37a47dc-6526-4995-9669-9aeb3fb606d6.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.665560439Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:52074","req_id":"08a680fd-eac0-4ed6-ba82-f2c518e63fc7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.857465522Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::2]:54833","req_id":"86f87cc3-534c-4ffe-96c1-ed63efc42bbe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45996,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:00:00.858124657Z","hostname":"oxz_internal_dns_5e875085-6c3b-435f-ae7c-56ed9b78f9bf","pid":5500,"peer_addr":"[fd00:1122:3344:101::2]:35465","req_id":"e118714e-e265-426a-8758-28c1b932149d","component":"dns"}