[ Mar 24 15:24:56 Enabled. ] [ Mar 24 15:24:56 Rereading configuration. ] [ Mar 24 15:24:56 Rereading configuration. ] [ Mar 24 15:24:57 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Mar 24 15:24:57 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.714233121Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.714426315Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.732045819Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.732065339Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.7321155Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:57.732401656Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.563584043Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:46756"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.564256477Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"910b06c1-d4d8-4ba2-a765-2f97c93532a5","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.564282248Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.580912071Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.580935181Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"910b06c1-d4d8-4ba2-a765-2f97c93532a5","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:24:58.580946811Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"910b06c1-d4d8-4ba2-a765-2f97c93532a5","remote_addr":"[fd00:1122:3344:3::2]:46756","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":17119,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15130,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:25:10.036243191Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:33815","req_id":"5003c534-e14b-4079-8037-d5cf4063981c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36480,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:25:10.041773668Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:44985","req_id":"3ac9589d-2604-45bc-96f6-e595baf09414","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3429,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:25:10.045794953Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:50361","req_id":"cd41e239-38c5-4e2a-b2c3-50461f708d66","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30113,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:26:49.33407631Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:63712","req_id":"40b6d118-8154-485b-a8de-f3d871604299","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54582,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:26:59.316242062Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:50669","req_id":"89547917-417e-4dd3-b961-28e7b40c9706","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11931,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:01.369288975Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:26488","req_id":"b215e0dc-33b0-4d8f-846a-8d9de0b067aa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18640,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:01.447468084Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:22560","req_id":"d4ed1a5c-9ebe-443d-b8c5-dcafadbb4874","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60525,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:01.479190207Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:6054","req_id":"75ef44df-a23b-4526-86a0-2d2692823ddf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34814,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 55,\n 101,\n 102,\n 56,\n 102,\n 97,\n 51,\n 45,\n 99,\n 55,\n 51,\n 52,\n 45,\n 52,\n 51,\n 52,\n 54,\n 45,\n 97,\n 101,\n 54,\n 101,\n 45,\n 100,\n 101,\n 100,\n 57,\n 100,\n 57,\n 48,\n 53,\n 56,\n 50,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:03.705981757Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:45124","req_id":"9edbe28d-ac72-4234-b787-5de5c52ed0fc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34814,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 55,\n 101,\n 102,\n 56,\n 102,\n 97,\n 51,\n 45,\n 99,\n 55,\n 51,\n 52,\n 45,\n 52,\n 51,\n 52,\n 54,\n 45,\n 97,\n 101,\n 54,\n 101,\n 45,\n 100,\n 101,\n 100,\n 57,\n 100,\n 57,\n 48,\n 53,\n 56,\n 50,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:03.706089279Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:60949","req_id":"bb06f0c8-cc84-43fb-8eef-097733afd7b1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:09.318273325Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:51634","req_id":"4abd1060-8183-4b3d-9369-0beb14e6896f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27727,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:16.699434185Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:27174","req_id":"bb5924b8-845c-4c70-bd27-3fff42c7bd1f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:16.895613037Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:44084"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:16.943696567Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"8e95f225-ee87-4e62-bd80-6c78e6f1678a","remote_addr":"[fd00:1122:3344:101::c]:44084","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":19976,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.138544151Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52162"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.140722787Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"60d457e6-63ca-403e-b2f0-3eef7b2a11cf","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.140786568Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"60d457e6-63ca-403e-b2f0-3eef7b2a11cf","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.140794149Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"60d457e6-63ca-403e-b2f0-3eef7b2a11cf","remote_addr":"[fd00:1122:3344:101::c]:52162","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":307,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5836,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:18.449498569Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:51806","req_id":"e1cef719-a15b-4e58-a591-a9e074c91df8","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.732733469Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36915"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:18.733091947Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"f1a4029c-455d-4bd4-b581-a45ad92acbd4","remote_addr":"[fd00:1122:3344:101::c]:36915","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":283,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9322,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:19.320081373Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:55762","req_id":"3fd9ee43-7c8a-4a03-b87a-8935929cd8a4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:19.70637566Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:25884","req_id":"4695b858-18b9-4c64-8e05-8f138802732f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:19.925017509Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:38679"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:19.925341827Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"cc52dbe0-7074-4883-9858-c20ecd80605a","remote_addr":"[fd00:1122:3344:101::c]:38679","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":248,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30591,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:21.562032418Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:44022","req_id":"6b1047f0-9b76-47cf-a13a-ed04425ec2ec","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16262,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:21.594381004Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:26904","req_id":"d8780b32-91e0-4543-b159-5776d4aa9f76","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:21.729486241Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:64663"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:21.729714356Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"6364d41e-42de-4d2f-959e-05e4a670b4bc","remote_addr":"[fd00:1122:3344:101::a]:64663","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":184,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:21.782241881Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:58321"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:27:21.782463506Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"c9057cca-34cc-4cef-a2a4-d7979533f13b","remote_addr":"[fd00:1122:3344:101::b]:58321","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":180,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:29.321801808Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:57852","req_id":"d13bfb80-174a-4028-b287-a51ec74132d3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17369,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:39.322590375Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:53313","req_id":"5f30e901-978e-4894-a8eb-7454c33339be","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21027,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:49.323879776Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:50451","req_id":"887f091a-2203-44fc-ba6b-a3bd22daacf9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20551,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:27:59.326296528Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:61086","req_id":"41f72910-61c9-4e81-a73d-3fd148987d5b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:00.878502095Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:45980"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:00.882227004Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"eec61f05-e5b5-4931-a9cc-a317d4381e05","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:00.882287415Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"eec61f05-e5b5-4931-a9cc-a317d4381e05","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:00.882300245Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"eec61f05-e5b5-4931-a9cc-a317d4381e05","remote_addr":"[fd00:1122:3344:101::c]:45980","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":312,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.131861221Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:58679"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.135358025Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"e0534390-d36a-4edf-94e9-da06bbe1f77f","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.135441197Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"e0534390-d36a-4edf-94e9-da06bbe1f77f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.135471268Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"e0534390-d36a-4edf-94e9-da06bbe1f77f","remote_addr":"[fd00:1122:3344:101::a]:58679","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":282,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.200489437Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50222"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.204064983Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"a6706c6e-ec68-4c47-bee9-02775214b76f","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.204102744Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"a6706c6e-ec68-4c47-bee9-02775214b76f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:01.204138925Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"a6706c6e-ec68-4c47-bee9-02775214b76f","remote_addr":"[fd00:1122:3344:101::b]:50222","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":392,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10122,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:02.577621179Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:43832","req_id":"703968d9-1196-4c0c-b363-336c558fc6b2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64962,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:02.579770914Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:24369","req_id":"ce9b81f5-8ccb-4035-87cc-366fa65d8980","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39535,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:02.623241987Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:18252","req_id":"fc1e65d2-aa02-4557-a576-474684cfc684","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.683179439Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:53806"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.683466955Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"c09350fe-033d-438c-bdba-b2202dd94bba","remote_addr":"[fd00:1122:3344:101::b]:53806","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":232,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.722262338Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:34438"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.722538884Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"e65a46b7-bf95-495b-b91c-18e61d127f5c","remote_addr":"[fd00:1122:3344:101::c]:34438","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.764470663Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:57930"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:28:02.764697368Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"1232ba45-d86c-4f72-b2e5-c49bb4aa3d01","remote_addr":"[fd00:1122:3344:101::a]:57930","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35380,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:09.327721678Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:62411","req_id":"ac016271-320d-4629-92a2-ba8b2e60efeb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48242,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:19.329494156Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:54493","req_id":"1b33a9d7-f59e-4c32-bd21-eceaff425464","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51694,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:29.331152603Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:58826","req_id":"c742286e-7345-4626-87dc-b4afd875146b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10514,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:39.333120454Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:57867","req_id":"9cd41f18-2343-4682-988b-cade0db046dd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9402,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:49.335265851Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:50368","req_id":"28e9b843-aa8d-4dae-890e-7e864ded3810","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18932,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:28:59.336619629Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:63768","req_id":"9744c471-fb9d-47c4-a218-b5691566962e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.872840226Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:32872"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.875120574Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"1b5ed980-b065-4b4d-8584-f9f8836e50cc","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.875143415Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"1b5ed980-b065-4b4d-8584-f9f8836e50cc","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.875176166Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"1b5ed980-b065-4b4d-8584-f9f8836e50cc","remote_addr":"[fd00:1122:3344:101::b]:32872","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":254,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.87821168Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:39292"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.882066892Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"8fe3f9a7-12ad-4e29-8bdf-c2cecd6d2afc","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.882212235Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"8fe3f9a7-12ad-4e29-8bdf-c2cecd6d2afc","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.882222135Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"8fe3f9a7-12ad-4e29-8bdf-c2cecd6d2afc","remote_addr":"[fd00:1122:3344:101::c]:39292","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":231,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.905026939Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:60814"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.908656996Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"4cb0796b-f9f3-4542-ac31-3833eea15086","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.90887369Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"4cb0796b-f9f3-4542-ac31-3833eea15086","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:00.908913451Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"4cb0796b-f9f3-4542-ac31-3833eea15086","remote_addr":"[fd00:1122:3344:101::a]:60814","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":376,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64645,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.143983759Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:59041","req_id":"777d955f-c636-4e65-95e2-43b3ab66b40e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53226,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.148404452Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:52783","req_id":"0e109aea-8749-4b84-9c60-9bd7b0dc079e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23256,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.152319226Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:56929","req_id":"77ada85c-4a87-4fe2-8a4e-79cde7c02e20","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3435,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.216668111Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:43408","req_id":"021bd3c8-d04f-42cb-a4b6-d9556565087e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60807,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.222184118Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:59450","req_id":"da1aa841-fb54-4faf-9e87-939c0c7be211","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.316158002Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45814"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.316391647Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"2c6d4614-25a6-489e-8ba0-0b7246924973","remote_addr":"[fd00:1122:3344:101::a]:45814","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":183,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.352181916Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:32859"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.352425182Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"ba0da70b-f684-40ff-9f8d-fd9a47b5691c","remote_addr":"[fd00:1122:3344:101::b]:32859","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":205,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34200,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:02.371852684Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:32729","req_id":"716ad1f8-db7c-4247-a373-719bd62e8529","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.462438826Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49164"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:29:02.462686061Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"GET","req_id":"28f2db4a-29d5-4faf-8162-ca8d435f290b","remote_addr":"[fd00:1122:3344:101::c]:49164","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":206,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46058,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:09.337598289Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:58472","req_id":"b68c3aa3-3976-4036-a182-08ea1883b5bc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58447,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:12.104175834Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:50318","req_id":"168764d0-b02c-4e0b-91fb-799008f3ceb6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53420,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:12.10825323Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:52131","req_id":"8f0c842d-0763-4acc-85ed-3e4f66828059","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48495,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:12.112310746Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:48462","req_id":"907ea688-ad07-4b32-95e6-23fa84f6eccd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4876,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:19.338633754Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:60685","req_id":"03453876-9b11-4fb9-8ca7-778da06424fe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49703,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:29.339119637Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:50131","req_id":"ebd32bde-dfdc-4a47-b16e-10700e595f1a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52787,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:39.340892335Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:55887","req_id":"b8277638-cb5b-4116-b916-b27a7b665d09","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5070,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:49.342620789Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:60872","req_id":"2a0bfb0c-6285-475d-a2ca-4e4694229fcc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49482,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:29:59.344192206Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:49720","req_id":"ec3f01ad-7b71-4580-baa8-c9d4ae18c35e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38417,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 55,\n 101,\n 102,\n 56,\n 102,\n 97,\n 51,\n 45,\n 99,\n 55,\n 51,\n 52,\n 45,\n 52,\n 51,\n 52,\n 54,\n 45,\n 97,\n 101,\n 54,\n 101,\n 45,\n 100,\n 101,\n 100,\n 57,\n 100,\n 57,\n 48,\n 53,\n 56,\n 50,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.256175227Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:46924","req_id":"790c0e30-322c-4172-8f65-3dbbd6b807d9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38417,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 55,\n 101,\n 102,\n 56,\n 102,\n 97,\n 51,\n 45,\n 99,\n 55,\n 51,\n 52,\n 45,\n 52,\n 51,\n 52,\n 54,\n 45,\n 97,\n 101,\n 54,\n 101,\n 45,\n 100,\n 101,\n 100,\n 57,\n 100,\n 57,\n 48,\n 53,\n 56,\n 50,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_97ef8fa3-c734-4346-ae6e-ded9d905827b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.256601496Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:33026","req_id":"8ed8ef08-dbef-4151-a272-a810629c851b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1803,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 97,\n 53,\n 56,\n 57,\n 56,\n 56,\n 50,\n 45,\n 49,\n 52,\n 54,\n 100,\n 45,\n 52,\n 51,\n 98,\n 54,\n 45,\n 56,\n 49,\n 50,\n 56,\n 45,\n 102,\n 54,\n 98,\n 98,\n 48,\n 52,\n 50,\n 51,\n 99,\n 99,\n 50,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.441818297Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::7]:52860","req_id":"5066fa72-f9eb-4478-b6fc-0c6c83a8732c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1803,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 97,\n 53,\n 56,\n 57,\n 56,\n 56,\n 50,\n 45,\n 49,\n 52,\n 54,\n 100,\n 45,\n 52,\n 51,\n 98,\n 54,\n 45,\n 56,\n 49,\n 50,\n 56,\n 45,\n 102,\n 54,\n 98,\n 98,\n 48,\n 52,\n 50,\n 51,\n 99,\n 99,\n 50,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ba589882-146d-43b6-8128-f6bb0423cc2a.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.442364108Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::7]:52703","req_id":"9548860f-9cf8-4177-846a-aeecab3faf97","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31700,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 50,\n 54,\n 49,\n 57,\n 54,\n 99,\n 99,\n 45,\n 98,\n 98,\n 57,\n 51,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 57,\n 57,\n 101,\n 56,\n 45,\n 53,\n 48,\n 48,\n 51,\n 97,\n 99,\n 97,\n 49,\n 99,\n 99,\n 50,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.476961682Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::3]:63921","req_id":"3a997f97-aeca-4502-b486-8ff7b02aefbb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31700,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 50,\n 54,\n 49,\n 57,\n 54,\n 99,\n 99,\n 45,\n 98,\n 98,\n 57,\n 51,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 57,\n 57,\n 101,\n 56,\n 45,\n 53,\n 48,\n 48,\n 51,\n 97,\n 99,\n 97,\n 49,\n 99,\n 99,\n 50,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b26196cc-bb93-4a1c-99e8-5003aca1cc22.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.477540084Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::3]:43170","req_id":"f06c0f64-f542-403b-a962-3d3d565c8529","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40419,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 97,\n 50,\n 101,\n 100,\n 97,\n 102,\n 102,\n 45,\n 54,\n 52,\n 52,\n 48,\n 45,\n 52,\n 48,\n 51,\n 57,\n 45,\n 98,\n 101,\n 102,\n 99,\n 45,\n 51,\n 48,\n 97,\n 51,\n 102,\n 55,\n 54,\n 98,\n 49,\n 53,\n 54,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.584254339Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::6]:59181","req_id":"6db638b0-56ba-40ea-9f4a-19d46a651761","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40419,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 97,\n 50,\n 101,\n 100,\n 97,\n 102,\n 102,\n 45,\n 54,\n 52,\n 52,\n 48,\n 45,\n 52,\n 48,\n 51,\n 57,\n 45,\n 98,\n 101,\n 102,\n 99,\n 45,\n 51,\n 48,\n 97,\n 51,\n 102,\n 55,\n 54,\n 98,\n 49,\n 53,\n 54,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ba2edaff-6440-4039-befc-30a3f76b1565.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.584843581Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::6]:54745","req_id":"e29102fc-9bbc-4c52-8901-89025cd510b9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19553,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 100,\n 51,\n 56,\n 57,\n 51,\n 98,\n 48,\n 45,\n 100,\n 57,\n 100,\n 100,\n 45,\n 52,\n 98,\n 100,\n 99,\n 45,\n 56,\n 99,\n 49,\n 48,\n 45,\n 57,\n 101,\n 55,\n 100,\n 98,\n 54,\n 53,\n 100,\n 98,\n 102,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.636539068Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::4]:46131","req_id":"5c014b5d-d116-4666-ad1d-364cac255943","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19553,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 100,\n 51,\n 56,\n 57,\n 51,\n 98,\n 48,\n 45,\n 100,\n 57,\n 100,\n 100,\n 45,\n 52,\n 98,\n 100,\n 99,\n 45,\n 56,\n 99,\n 49,\n 48,\n 45,\n 57,\n 101,\n 55,\n 100,\n 98,\n 54,\n 53,\n 100,\n 98,\n 102,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_cd3893b0-d9dd-4bdc-8c10-9e7db65dbfa4.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.637010768Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::4]:57793","req_id":"d85b0457-514c-49d4-bfd1-8d4a479c9b6c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51319,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 49,\n 49,\n 55,\n 51,\n 57,\n 57,\n 49,\n 45,\n 97,\n 48,\n 48,\n 50,\n 45,\n 52,\n 97,\n 51,\n 50,\n 45,\n 56,\n 100,\n 98,\n 56,\n 45,\n 52,\n 99,\n 100,\n 98,\n 100,\n 49,\n 101,\n 50,\n 48,\n 55,\n 54,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.645217392Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::5]:36881","req_id":"b8c39156-e9bb-4c4f-a15c-c70adbd9a1c0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51319,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 49,\n 49,\n 55,\n 51,\n 57,\n 57,\n 49,\n 45,\n 97,\n 48,\n 48,\n 50,\n 45,\n 52,\n 97,\n 51,\n 50,\n 45,\n 56,\n 100,\n 98,\n 56,\n 45,\n 52,\n 99,\n 100,\n 98,\n 100,\n 49,\n 101,\n 50,\n 48,\n 55,\n 54,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_41173991-a002-4a32-8db8-4cdbd1e2076d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:00.645645971Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::5]:43753","req_id":"1d4e9527-cb78-45ce-9af6-d1e16b6e2c9f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.873917715Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:48207"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.877685815Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"de2461f8-043f-4225-8cce-f7b79eaf46b0","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.877714526Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"de2461f8-043f-4225-8cce-f7b79eaf46b0","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.877723856Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"de2461f8-043f-4225-8cce-f7b79eaf46b0","remote_addr":"[fd00:1122:3344:101::b]:48207","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":356,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.878030712Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:44878"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.882339104Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"5ce7adf1-dac9-42a1-a114-290c2f9d3025","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.882371924Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"5ce7adf1-dac9-42a1-a114-290c2f9d3025","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.882750382Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"5ce7adf1-dac9-42a1-a114-290c2f9d3025","remote_addr":"[fd00:1122:3344:101::c]:44878","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":871,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.905374262Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:37514"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.908926208Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"89b07e26-e47e-4624-b873-4212fd9a9660","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.908973589Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"new_generation":1,"req_id":"89b07e26-e47e-4624-b873-4212fd9a9660","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T15:30:00.908987079Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"uri":"/config","method":"PUT","req_id":"89b07e26-e47e-4624-b873-4212fd9a9660","remote_addr":"[fd00:1122:3344:101::a]:37514","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":294,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27023,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ab9a1c21-d4d2-43f1-8da7-049b0c75719b.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ab9a1c21-d4d2-43f1-8da7-049b0c75719b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 97,\n 98,\n 57,\n 97,\n 49,\n 99,\n 50,\n 49,\n 45,\n 100,\n 52,\n 100,\n 50,\n 45,\n 52,\n 51,\n 102,\n 49,\n 45,\n 56,\n 100,\n 97,\n 55,\n 45,\n 48,\n 52,\n 57,\n 98,\n 48,\n 99,\n 55,\n 53,\n 55,\n 49,\n 57,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ab9a1c21-d4d2-43f1-8da7-049b0c75719b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:01.087464936Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:57363","req_id":"da5c9c73-d56e-4938-9337-8f545bed496c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55062,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:01.438058706Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::2]:64267","req_id":"ff78dd11-7d38-41c3-9ae9-62c757369632","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55062,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:01.439051057Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::2]:33624","req_id":"1819aded-e459-4047-847e-e7136e7d6304","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31760,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:02.58678044Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:50871","req_id":"967bce67-a28f-432f-b96d-7922e9439070","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43025,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T15:30:02.59478494Z","hostname":"oxz_internal_dns_328b7603-ac6d-4fa7-af51-ab33f8481639","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:7436","req_id":"8d7cbc33-7667-4667-9edd-bdb5ea1b74af","component":"dns"}