[ May 1 20:10:06 Enabled. ] [ May 1 20:10:06 Rereading configuration. ] [ May 1 20:10:07 Rereading configuration. ] [ May 1 20:10:11 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 1 20:10:11 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.090054099Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.090193542Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.107177133Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.107196074Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.107264795Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.107478849Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.110011687Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:34479"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.111133818Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"d4098736-2313-476d-8550-886af0b8d87e","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.111148748Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.274374765Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.274401845Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"d4098736-2313-476d-8550-886af0b8d87e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.274456526Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"PUT","req_id":"d4098736-2313-476d-8550-886af0b8d87e","remote_addr":"[fd00:1122:3344:3::2]:34479","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":163885,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34251,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.320507944Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:33497","req_id":"a115af68-438d-42ca-af02-3d31023301c1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50955,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.326276594Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:50452","req_id":"cd2fc182-68d5-4645-b418-19ee52c10e14","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.330302759Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:53840","req_id":"8f773da4-266e-4580-80f8-95c6e8c0bfa9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60531,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.049587869Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:50897","req_id":"6fd5b760-e519-4e8d-a18b-4e4b6b3f8996","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51987,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.273727716Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:57815","req_id":"f243e834-48f9-49e8-a52c-acccc522b367","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28316,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.289723043Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:50166","req_id":"09b47d9f-7ed7-4815-b19f-e9f72f77336c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47534,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:30.741149971Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:60218","req_id":"4af58ba7-bf89-416d-aeee-5a2881da9102","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 355,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 100,\n 54,\n 98,\n 99,\n 57,\n 98,\n 51,\n 45,\n 49,\n 50,\n 50,\n 100,\n 45,\n 52,\n 97,\n 49,\n 98,\n 45,\n 98,\n 102,\n 50,\n 98,\n 45,\n 98,\n 101,\n 100,\n 99,\n 51,\n 54,\n 50,\n 98,\n 102,\n 49,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:34.546866412Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::e]:58017","req_id":"ba76a888-2e2f-408c-bf0b-7355ae5e07de","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 355,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 100,\n 54,\n 98,\n 99,\n 57,\n 98,\n 51,\n 45,\n 49,\n 50,\n 50,\n 100,\n 45,\n 52,\n 97,\n 49,\n 98,\n 45,\n 98,\n 102,\n 50,\n 98,\n 45,\n 98,\n 101,\n 100,\n 99,\n 51,\n 54,\n 50,\n 98,\n 102,\n 49,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:34.547698008Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::e]:57496","req_id":"4cec518b-63c7-4d4c-baf9-2d719431b8a9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41769,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.518534001Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::c]:42659","req_id":"b1fffc39-0bbc-4500-8854-3694ec9e6f50","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8855,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.568177339Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::a]:54198","req_id":"d4f00664-fb54-4d3a-b8e6-5b4572a471b2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61554,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.728762306Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:63670","req_id":"db22789c-543f-406c-adcf-e957ce98c2f1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21778,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:41.096667973Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::b]:36645","req_id":"c01f9b8b-07a2-49c7-b277-498c4f9b443f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.294195606Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:61037"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.354979496Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"35f5ef0d-ad1a-4579-9dd5-7ffbce52a371","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.355048067Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"35f5ef0d-ad1a-4579-9dd5-7ffbce52a371","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.355126078Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"PUT","req_id":"35f5ef0d-ad1a-4579-9dd5-7ffbce52a371","remote_addr":"[fd00:1122:3344:101::a]:61037","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":28755,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.487343249Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:55161"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.487445631Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:56387"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.487456111Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:53036"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.49800667Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"fd190e44-2dad-4033-ad2c-d2e26276b5d6","remote_addr":"[fd00:1122:3344:101::b]:56387","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":10035,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.498078441Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"16b1e9f2-9ef3-4329-9f70-ca75e235d153","remote_addr":"[fd00:1122:3344:101::a]:55161","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":10055,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.498135833Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"6837a10f-eb2b-4632-af29-b614aed81e01","remote_addr":"[fd00:1122:3344:101::c]:53036","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":10035,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30510,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:43.756401264Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::a]:23074","req_id":"fab65535-399c-4591-bfa1-e0b50eb68631","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:43.888519223Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:35411"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:43.888843449Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"cffc5efb-c725-4f46-bf9c-493c82db05c9","remote_addr":"[fd00:1122:3344:101::a]:35411","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":255,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42613,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:44.535659879Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::a]:35508","req_id":"a7235252-4def-4a3d-8411-b047e442dabf","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:44.7514583Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:37824"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:44.751759775Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"16f4feb1-abca-4310-8f46-0d3524881ffb","remote_addr":"[fd00:1122:3344:101::a]:37824","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":244,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11139,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:45.218557942Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::a]:37445","req_id":"e7e5690a-d658-4108-9b76-a897783cd529","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:45.443294042Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:35874"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:45.443561317Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"b17716ad-9181-451a-ad5b-b7f106179484","remote_addr":"[fd00:1122:3344:101::a]:35874","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38263,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:50.73115773Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:61069","req_id":"57f39c78-7ed9-4292-ba60-9d6affb1e034","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15775,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:51.295380649Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::c]:64110","req_id":"3fbfc822-109f-4bd6-9dd3-377a01663839","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:51.412462553Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52880"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:51.412794659Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"745c5d5d-024d-4d45-87a3-e53929702ccb","remote_addr":"[fd00:1122:3344:101::c]:52880","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":255,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3782,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:51.875553058Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::b]:34120","req_id":"0951c443-0c4e-4194-94dc-c6fd60758d54","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:52.076660921Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:62846"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:52.076947886Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"0dc14a94-06a5-4086-8935-6b36045d712f","remote_addr":"[fd00:1122:3344:101::b]:62846","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":229,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39537,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:00.732529493Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:65208","req_id":"3c15c7f4-a4ef-4252-83bf-ae033c3a921c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51359,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:10.734445028Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:55642","req_id":"b1f41f1f-4ebd-4d50-ba61-991e8ee559aa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30559,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:20.73628279Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:52253","req_id":"dffcd500-1e6c-4edf-9ede-079ccc538c9f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32271,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.715237753Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:37120","req_id":"78e2b5ab-e682-4399-b16a-fe75ebb65672","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20704,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.719769449Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:34093","req_id":"9b27ce2f-f771-42b3-ba9a-e024b9167f08","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51739,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.72405726Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:64623","req_id":"9a66947d-128d-43d8-9f13-9c13bc0879c7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50633,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:30.738297936Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:64450","req_id":"4296f127-4754-4a46-bdf8-ee9b8513b687","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.286166695Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:32943"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288783145Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"b1406078-b5d7-4003-a0e3-1c7a8c15d198","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288815295Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"b1406078-b5d7-4003-a0e3-1c7a8c15d198","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288847066Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"PUT","req_id":"b1406078-b5d7-4003-a0e3-1c7a8c15d198","remote_addr":"[fd00:1122:3344:101::a]:32943","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":223,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.615543643Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:56045"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.618200383Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"be982558-98e7-422c-9375-bce0be0a1b63","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.618248684Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"be982558-98e7-422c-9375-bce0be0a1b63","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.618257334Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"PUT","req_id":"be982558-98e7-422c-9375-bce0be0a1b63","remote_addr":"[fd00:1122:3344:101::c]:56045","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":781,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48152,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:40.74020069Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:62267","req_id":"6c994dde-67f7-4eec-abe2-0d41bdab9612","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.028513802Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:34209"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.031018979Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"d4fdd19b-dccd-4dea-be6b-c5ab1105638f","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.0310721Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"new_generation":1,"req_id":"d4fdd19b-dccd-4dea-be6b-c5ab1105638f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.03108222Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"PUT","req_id":"d4fdd19b-dccd-4dea-be6b-c5ab1105638f","remote_addr":"[fd00:1122:3344:101::b]:34209","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":244,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48913,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.795899692Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::a]:62641","req_id":"91cb388f-9ea5-43ad-855c-d6c095712c35","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1263,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.805421732Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::c]:2195","req_id":"3c0b4255-8ca9-476c-9226-4a99c02f66e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40924,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.806054624Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::b]:28476","req_id":"53e16095-2bf3-47d5-b729-b3807f408b16","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.920621451Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:35850"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.920906476Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"b02656a2-f3f4-431c-b29e-bbd33f234498","remote_addr":"[fd00:1122:3344:101::c]:35850","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":245,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.989405981Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:54788"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.989710927Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"da025b92-d26f-4d0e-ae5e-14e5f8754331","remote_addr":"[fd00:1122:3344:101::a]:54788","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":242,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:42.028931779Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:46121"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:42.029159893Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"uri":"/config","method":"GET","req_id":"52ba7dd6-73aa-4fa3-89e6-ee887de22491","remote_addr":"[fd00:1122:3344:101::b]:46121","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":198,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33758,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:50.742302578Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::d]:56033","req_id":"88fbe5b4-7534-4d67-b310-a382b77d546c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 53,\n 98,\n 49,\n 51,\n 98,\n 53,\n 54,\n 55,\n 45,\n 50,\n 100,\n 49,\n 57,\n 45,\n 52,\n 100,\n 54,\n 102,\n 45,\n 98,\n 55,\n 56,\n 52,\n 45,\n 97,\n 52,\n 102,\n 54,\n 57,\n 51,\n 57,\n 52,\n 102,\n 100,\n 56,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:15:00.123722181Z","hostname":"oxz_internal_dns_0b164eb8-930c-43b8-9cd3-8c7585432c7e","pid":5559,"peer_addr":"[fd00:1122:3344:101::1b]:55532","req_id":"5e24594d-4bab-4346-8227-49d4769279ff","component":"dns"}